Skip to content

Bring projects, wikis, and teams together with AI. AppFlowy is the AI collaborative workspace where you achieve more without losing control of your data. The leading open source Notion alternative.

chore: improvements to caching strategies (#5488)

Kilu.He86696b271e35
diff --git a/frontend/appflowy_web_app/src/application/database-yjs/selector.ts b/frontend/appflowy_web_app/src/application/database-yjs/selector.ts
index 49fceb885c..884c58516d 100644
--- a/frontend/appflowy_web_app/src/application/database-yjs/selector.ts
+++ b/frontend/appflowy_web_app/src/application/database-yjs/selector.ts
@@ -71,7 +71,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {
const [key] = viewItem;
const view = folderViews?.get(key);
- console.log('view', view?.get(YjsFolderKey.bid), iidIndex);
if (
visibleViewsId.includes(key) &&
view &&
@@ -81,7 +80,6 @@ export function useDatabaseViewsSelector(iidIndex: string) {
}
}
- console.log('viewsId', viewsId);
setViewIds(viewsId);
};
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.ts b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.ts
new file mode 100644
index 0000000000..9a7e3fcb9d
--- /dev/null
+++ b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/cache.test.ts
@@ -0,0 +1,310 @@
+import { CollabType } from '@/application/collab.type';
+import * as Y from 'yjs';
+import { withTestingYDoc } from '@/application/slate-yjs/__tests__/withTestingYjsEditor';
+import { expect } from '@jest/globals';
+import { getCollab, batchCollab, collabTypeToDBType } from '../cache';
+import { applyYDoc } from '@/application/ydoc/apply';
+import { getCollabDBName, openCollabDB } from '../cache/db';
+import { StrategyType } from '../cache/types';
+
+jest.mock('@/application/ydoc/apply', () => ({
+ applyYDoc: jest.fn(),
+}));
+jest.mock('../cache/db', () => ({
+ openCollabDB: jest.fn(),
+ getCollabDBName: jest.fn(),
+}));
+
+const emptyDoc = new Y.Doc();
+const normalDoc = withTestingYDoc('1');
+const mockFetcher = jest.fn();
+const mockBatchFetcher = jest.fn();
+
+describe('Cache functions', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ describe('getCollab', () => {
+ describe('with CACHE_ONLY strategy', () => {
+ it('should throw error when no cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ await expect(
+ getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.CACHE_ONLY
+ )
+ ).rejects.toThrow('No cache found');
+ });
+ it('should fetch collab with CACHE_ONLY strategy and existing cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ const result = await getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.CACHE_ONLY
+ );
+
+ expect(result).toBe(normalDoc);
+ expect(mockFetcher).not.toHaveBeenCalled();
+ expect(applyYDoc).not.toHaveBeenCalled();
+ });
+ });
+
+ describe('with CACHE_FIRST strategy', () => {
+ it('should fetch collab with CACHE_FIRST strategy and existing cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+ const result = await getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.CACHE_FIRST
+ );
+
+ expect(result).toBe(normalDoc);
+ expect(mockFetcher).not.toHaveBeenCalled();
+ expect(applyYDoc).not.toHaveBeenCalled();
+ });
+
+ it('should fetch collab with CACHE_FIRST strategy and no cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+ const result = await getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.CACHE_FIRST
+ );
+
+ expect(result).toBe(emptyDoc);
+ expect(mockFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+ });
+
+ describe('with CACHE_AND_NETWORK strategy', () => {
+ it('should fetch collab with CACHE_AND_NETWORK strategy and existing cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+ const result = await getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.CACHE_AND_NETWORK
+ );
+
+ expect(result).toBe(normalDoc);
+ expect(mockFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+
+ it('should fetch collab with CACHE_AND_NETWORK strategy and no cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+ const result = await getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.CACHE_AND_NETWORK
+ );
+
+ expect(result).toBe(emptyDoc);
+ expect(mockFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+ });
+
+ describe('with default strategy', () => {
+ it('should fetch collab with default strategy', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ mockFetcher.mockResolvedValue({ state: new Uint8Array() });
+
+ const result = await getCollab(
+ mockFetcher,
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ StrategyType.NETWORK_ONLY
+ );
+
+ expect(result).toBe(normalDoc);
+ expect(mockFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+ });
+ });
+
+ describe('batchCollab', () => {
+ describe('with CACHE_ONLY strategy', () => {
+ it('should batch fetch collabs with CACHE_ONLY strategy and no cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ await expect(
+ batchCollab(
+ mockBatchFetcher,
+ [
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ ],
+ StrategyType.CACHE_ONLY
+ )
+ ).rejects.toThrow('No cache found');
+ });
+
+ it('should batch fetch collabs with CACHE_ONLY strategy and existing cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ await batchCollab(
+ mockBatchFetcher,
+ [
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ ],
+ StrategyType.CACHE_ONLY
+ );
+
+ expect(mockBatchFetcher).not.toHaveBeenCalled();
+ });
+ });
+
+ describe('with CACHE_FIRST strategy', () => {
+ it('should batch fetch collabs with CACHE_FIRST strategy and existing cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+
+ await batchCollab(
+ mockBatchFetcher,
+ [
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ ],
+ StrategyType.CACHE_FIRST
+ );
+
+ expect(mockBatchFetcher).not.toHaveBeenCalled();
+ });
+
+ it('should batch fetch collabs with CACHE_FIRST strategy and no cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+ mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
+
+ await batchCollab(
+ mockBatchFetcher,
+ [
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ ],
+ StrategyType.CACHE_FIRST
+ );
+
+ expect(mockBatchFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+ });
+
+ describe('with CACHE_AND_NETWORK strategy', () => {
+ it('should batch fetch collabs with CACHE_AND_NETWORK strategy', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(normalDoc);
+
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+ mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
+
+ await batchCollab(
+ mockBatchFetcher,
+ [
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ ],
+ StrategyType.CACHE_AND_NETWORK
+ );
+
+ expect(mockBatchFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+
+ it('should batch fetch collabs with CACHE_AND_NETWORK strategy and no cache', async () => {
+ (openCollabDB as jest.Mock).mockResolvedValue(emptyDoc);
+
+ (getCollabDBName as jest.Mock).mockReturnValue('testDB');
+ mockBatchFetcher.mockResolvedValue({ id1: [1, 2, 3] });
+
+ await batchCollab(
+ mockBatchFetcher,
+ [
+ {
+ collabId: 'id1',
+ collabType: CollabType.Document,
+ },
+ ],
+ StrategyType.CACHE_AND_NETWORK
+ );
+
+ expect(mockBatchFetcher).toHaveBeenCalled();
+ expect(applyYDoc).toHaveBeenCalled();
+ });
+ });
+ });
+});
+
+describe('collabTypeToDBType', () => {
+ it('should return correct DB type', () => {
+ expect(collabTypeToDBType(CollabType.Document)).toBe('document');
+ expect(collabTypeToDBType(CollabType.Folder)).toBe('folder');
+ expect(collabTypeToDBType(CollabType.Database)).toBe('database');
+ expect(collabTypeToDBType(CollabType.WorkspaceDatabase)).toBe('databases');
+ expect(collabTypeToDBType(CollabType.DatabaseRow)).toBe('database_row');
+ expect(collabTypeToDBType(CollabType.UserAwareness)).toBe('user_awareness');
+ expect(collabTypeToDBType(CollabType.Empty)).toBe('');
+ });
+});
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.ts b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.ts
new file mode 100644
index 0000000000..afdc418c2a
--- /dev/null
+++ b/frontend/appflowy_web_app/src/application/services/js-services/__tests__/fetch.test.ts
@@ -0,0 +1,57 @@
+import { expect } from '@jest/globals';
+import { fetchCollab, batchFetchCollab } from '../fetch';
+import { CollabType } from '@/application/collab.type';
+import { APIService } from '@/application/services/js-services/wasm';
+
+jest.mock('@/application/services/js-services/wasm', () => {
+ return {
+ APIService: {
+ getCollab: jest.fn(),
+ batchGetCollab: jest.fn(),
+ },
+ };
+});
+
+describe('Collab fetch functions with deduplication', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ describe('fetchCollab', () => {
+ it('should fetch collab without duplicating requests', async () => {
+ const workspaceId = 'workspace1';
+ const id = 'id1';
+ const type = CollabType.Document;
+ const mockResponse = { data: 'mockData' };
+
+ (APIService.getCollab as jest.Mock).mockResolvedValue(mockResponse);
+
+ const result1 = fetchCollab(workspaceId, id, type);
+ const result2 = fetchCollab(workspaceId, id, type);
+
+ expect(result1).toBe(result2);
+ await expect(result1).resolves.toEqual(mockResponse);
+ expect(APIService.getCollab).toHaveBeenCalledTimes(1);
+ });
+ });
+
+ describe('batchFetchCollab', () => {
+ it('should batch fetch collabs without duplicating requests', async () => {
+ const workspaceId = 'workspace1';
+ const params = [
+ { collabId: 'id1', collabType: CollabType.Document },
+ { collabId: 'id2', collabType: CollabType.Folder },
+ ];
+ const mockResponse = { data: 'mockData' };
+
+ (APIService.batchGetCollab as jest.Mock).mockResolvedValue(mockResponse);
+
+ const result1 = batchFetchCollab(workspaceId, params);
+ const result2 = batchFetchCollab(workspaceId, params);
+
+ expect(result1).toBe(result2);
+ await expect(result1).resolves.toEqual(mockResponse);
+ expect(APIService.batchGetCollab).toHaveBeenCalledTimes(1);
+ });
+ });
+});
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts
index 7119497775..7f80c9f871 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/auth.service.ts
@@ -1,8 +1,8 @@
import { AuthService } from '@/application/services/services.type';
import { ProviderType, SignUpWithEmailPasswordParams } from '@/application/user.type';
import { APIService } from 'src/application/services/js-services/wasm';
-import { signInSuccess } from '@/application/services/js-services/storage/auth';
-import { invalidToken } from '@/application/services/js-services/storage';
+import { signInSuccess } from '@/application/services/js-services/session/auth';
+import { invalidToken } from 'src/application/services/js-services/session';
import { afterSignInDecorator } from '@/application/services/js-services/decorator';
export class JSAuthService implements AuthService {
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/db/index.ts b/frontend/appflowy_web_app/src/application/services/js-services/cache/db.ts
similarity index 76%
rename from frontend/appflowy_web_app/src/application/services/js-services/db/index.ts
rename to frontend/appflowy_web_app/src/application/services/js-services/cache/db.ts
index f17ac50531..a4d888498b 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/db/index.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/cache/db.ts
@@ -1,9 +1,10 @@
import { YDoc } from '@/application/collab.type';
import { databasePrefix } from '@/application/constants';
-import { getAuthInfo } from '@/application/services/js-services/storage';
import { IndexeddbPersistence } from 'y-indexeddb';
import * as Y from 'yjs';
+const openedSet = new Set<string>();
+
/**
* Open the collaboration database, and return a function to close it
*/
@@ -19,6 +20,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {
});
provider.on('synced', () => {
+ if (!openedSet.has(name)) {
+ openedSet.add(name);
+ }
+
resolve(true);
});
@@ -27,9 +32,10 @@ export async function openCollabDB(docName: string): Promise<YDoc> {
return doc as YDoc;
}
-export function getDBName(id: string, type: string) {
- const { uuid } = getAuthInfo() || {};
+export function getCollabDBName(id: string, type: string, uuid?: string) {
+ if (!uuid) {
+ return `${type}_${id}`;
+ }
- if (!uuid) throw new Error('No user found');
return `${uuid}_${type}_${id}`;
}
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts b/frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts
new file mode 100644
index 0000000000..1f7fe670f1
--- /dev/null
+++ b/frontend/appflowy_web_app/src/application/services/js-services/cache/index.ts
@@ -0,0 +1,165 @@
+import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
+import { applyYDoc } from '@/application/ydoc/apply';
+import { getCollabDBName, openCollabDB } from './db';
+import { Fetcher, StrategyType } from './types';
+
+export function collabTypeToDBType(type: CollabType) {
+ switch (type) {
+ case CollabType.Folder:
+ return 'folder';
+ case CollabType.Document:
+ return 'document';
+ case CollabType.Database:
+ return 'database';
+ case CollabType.WorkspaceDatabase:
+ return 'databases';
+ case CollabType.DatabaseRow:
+ return 'database_row';
+ case CollabType.UserAwareness:
+ return 'user_awareness';
+ default:
+ return '';
+ }
+}
+
+const collabSharedRootKeyMap = {
+ [CollabType.Folder]: YjsEditorKey.folder,
+ [CollabType.Document]: YjsEditorKey.document,
+ [CollabType.Database]: YjsEditorKey.database,
+ [CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,
+ [CollabType.DatabaseRow]: YjsEditorKey.database_row,
+ [CollabType.UserAwareness]: YjsEditorKey.user_awareness,
+ [CollabType.Empty]: YjsEditorKey.empty,
+};
+
+export function hasCache(doc: YDoc, type: CollabType) {
+ const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
+
+ return data.has(collabSharedRootKeyMap[type] as string);
+}
+
+export async function getCollab(
+ fetcher: Fetcher<{
+ state: Uint8Array;
+ }>,
+ {
+ collabId,
+ collabType,
+ uuid,
+ }: {
+ uuid?: string;
+ collabId: string;
+ collabType: CollabType;
+ },
+ strategy: StrategyType = StrategyType.CACHE_AND_NETWORK
+) {
+ const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);
+ const collab = await openCollabDB(name);
+ const exist = hasCache(collab, collabType);
+
+ switch (strategy) {
+ case StrategyType.CACHE_ONLY: {
+ if (!exist) {
+ throw new Error('No cache found');
+ }
+
+ return collab;
+ }
+
+ case StrategyType.CACHE_FIRST: {
+ if (!exist) {
+ await revalidateCollab(fetcher, collab);
+ }
+
+ return collab;
+ }
+
+ case StrategyType.CACHE_AND_NETWORK: {
+ if (!exist) {
+ await revalidateCollab(fetcher, collab);
+ } else {
+ void revalidateCollab(fetcher, collab);
+ }
+
+ return collab;
+ }
+
+ default: {
+ await revalidateCollab(fetcher, collab);
+
+ return collab;
+ }
+ }
+}
+
+async function revalidateCollab(
+ fetcher: Fetcher<{
+ state: Uint8Array;
+ }>,
+ collab: YDoc
+) {
+ const { state } = await fetcher();
+
+ applyYDoc(collab, state);
+}
+
+export async function batchCollab(
+ batchFetcher: Fetcher<Record<string, number[]>>,
+ collabs: {
+ collabId: string;
+ collabType: CollabType;
+ uuid?: string;
+ }[],
+ strategy: StrategyType = StrategyType.CACHE_AND_NETWORK,
+ itemCallback?: (id: string, doc: YDoc) => void
+) {
+ const collabMap = new Map<string, YDoc>();
+
+ for (const { collabId, collabType, uuid } of collabs) {
+ const name = getCollabDBName(collabId, collabTypeToDBType(collabType), uuid);
+ const collab = await openCollabDB(name);
+ const exist = hasCache(collab, collabType);
+
+ collabMap.set(collabId, collab);
+ if (exist) {
+ itemCallback?.(collabId, collab);
+ }
+ }
+
+ const notCacheIds = collabs.filter(({ collabId, collabType }) => {
+ const id = collabMap.get(collabId);
+
+ if (!id) return false;
+
+ return !hasCache(id, collabType);
+ });
+
+ if (strategy === StrategyType.CACHE_ONLY) {
+ if (notCacheIds.length > 0) {
+ throw new Error('No cache found');
+ }
+
+ return;
+ }
+
+ if (strategy === StrategyType.CACHE_FIRST && notCacheIds.length === 0) {
+ return;
+ }
+
+ const states = await batchFetcher();
+
+ for (const [collabId, data] of Object.entries(states)) {
+ const info = collabs.find((item) => item.collabId === collabId);
+ const collab = collabMap.get(collabId);
+
+ if (!info || !collab) {
+ continue;
+ }
+
+ const state = new Uint8Array(data);
+
+ applyYDoc(collab, state);
+
+ itemCallback?.(collabId, collab);
+ }
+}
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts b/frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts
new file mode 100644
index 0000000000..1c1a949723
--- /dev/null
+++ b/frontend/appflowy_web_app/src/application/services/js-services/cache/types.ts
@@ -0,0 +1,12 @@
+export enum StrategyType {
+ // Cache only: return the cache if it exists, otherwise throw an error
+ CACHE_ONLY = 'CACHE_ONLY',
+ // Cache first: return the cache if it exists, otherwise fetch from the network
+ CACHE_FIRST = 'CACHE_FIRST',
+ // Cache and network: return the cache if it exists, otherwise fetch from the network and update the cache
+ CACHE_AND_NETWORK = 'CACHE_AND_NETWORK',
+ // Network only: fetch from the network and update the cache
+ NETWORK_ONLY = 'NETWORK_ONLY',
+}
+
+export type Fetcher<T> = () => Promise<T>;
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts
index b1dcfe8d0f..cf29b221dd 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/database.service.ts
@@ -1,16 +1,16 @@
import { CollabType, YDatabase, YDoc, YjsDatabaseKey, YjsEditorKey } from '@/application/collab.type';
-import {
- batchCollabs,
- getCollabStorage,
- getCollabStorageWithAPICall,
- getCurrentWorkspace,
-} from '@/application/services/js-services/storage';
+import { batchCollab, getCollab } from '@/application/services/js-services/cache';
+import { StrategyType } from '@/application/services/js-services/cache/types';
+import { batchFetchCollab, fetchCollab } from '@/application/services/js-services/fetch';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
import { DatabaseService } from '@/application/services/services.type';
import * as Y from 'yjs';
export class JSDatabaseService implements DatabaseService {
private loadedDatabaseId: Set<string> = new Set();
+ private loadedWorkspaceId: Set<string> = new Set();
+
private cacheDatabaseRowDocMap: Map<string, Y.Doc> = new Map();
constructor() {
@@ -28,22 +28,30 @@ export class JSDatabaseService implements DatabaseService {
throw new Error('Workspace database not found');
}
- const workspaceDatabase = await getCollabStorageWithAPICall(
- workspace.id,
- workspace.workspaceDatabaseId,
- CollabType.WorkspaceDatabase
+ const isLoaded = this.loadedWorkspaceId.has(workspace.id);
+
+ const workspaceDatabase = await getCollab(
+ () => {
+ return fetchCollab(workspace.id, workspace.workspaceDatabaseId, CollabType.WorkspaceDatabase);
+ },
+ {
+ collabId: workspace.workspaceDatabaseId,
+ collabType: CollabType.WorkspaceDatabase,
+ },
+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
);
+ if (!isLoaded) {
+ this.loadedWorkspaceId.add(workspace.id);
+ }
+
return workspaceDatabase.getMap(YjsEditorKey.data_section).get(YjsEditorKey.workspace_database).toJSON() as {
views: string[];
database_id: string;
}[];
}
- async openDatabase(
- databaseId: string,
- rowIds?: string[]
- ): Promise<{
+ async openDatabase(databaseId: string): Promise<{
databaseDoc: YDoc;
rows: Y.Map<YDoc>;
}> {
@@ -68,13 +76,18 @@ export class JSDatabaseService implements DatabaseService {
const rowsFolder: Y.Map<YDoc> = rootRowsDoc.getMap();
- let databaseDoc: YDoc | undefined = undefined;
+ const databaseDoc = await getCollab(
+ () => {
+ return fetchCollab(workspaceId, databaseId, CollabType.Database);
+ },
+ {
+ collabId: databaseId,
+ collabType: CollabType.Database,
+ },
+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
+ );
- if (isLoaded) {
- databaseDoc = (await getCollabStorage(databaseId, CollabType.Database)).doc;
- } else {
- databaseDoc = await getCollabStorageWithAPICall(workspaceId, databaseId, CollabType.Database);
- }
+ if (!isLoaded) this.loadedDatabaseId.add(databaseId);
const database = databaseDoc.getMap(YjsEditorKey.data_section)?.get(YjsEditorKey.database) as YDatabase;
const viewId = database.get(YjsDatabaseKey.metas)?.get(YjsDatabaseKey.iid)?.toString();
@@ -87,47 +100,50 @@ export class JSDatabaseService implements DatabaseService {
throw new Error('Database rows not found');
}
- const ids = rowIds ? rowIds : rowOrdersIds.map((item) => item.id);
-
- if (isLoaded) {
- for (const id of ids) {
- const { doc } = await getCollabStorage(id, CollabType.DatabaseRow);
-
+ const rowsParams = rowOrdersIds.map((item) => ({
+ collabId: item.id,
+ collabType: CollabType.DatabaseRow,
+ }));
+
+ void batchCollab(
+ () => {
+ return batchFetchCollab(workspaceId, rowsParams);
+ },
+ rowsParams,
+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK,
+ (id: string, doc: YDoc) => {
if (!rowsFolder.has(id)) {
rowsFolder.set(id, doc);
}
}
- } else {
- void this.loadDatabaseRows(workspaceId, ids, (id, row) => {
- if (!rowsFolder.has(id)) {
- rowsFolder.set(id, row);
- }
- });
- }
+ );
- this.loadedDatabaseId.add(databaseId);
-
- if (!rowIds) {
- // Update rows if new rows are added
- rowOrders?.observe((event) => {
- if (event.changes.added.size > 0) {
- const rowIds = rowOrders.toJSON() as {
- id: string;
- }[];
-
- console.log('Update rows', rowIds);
- void this.loadDatabaseRows(
- workspaceId,
- rowIds.map((item) => item.id),
- (rowId: string, rowDoc) => {
- if (!rowsFolder.has(rowId)) {
- rowsFolder.set(rowId, rowDoc);
- }
+ // Update rows if there are new rows added after the database has been loaded
+ rowOrders?.observe((event) => {
+ if (event.changes.added.size > 0) {
+ const rowIds = rowOrders.toJSON() as {
+ id: string;
+ }[];
+
+ const params = rowIds.map((item) => ({
+ collabId: item.id,
+ collabType: CollabType.DatabaseRow,
+ }));
+
+ void batchCollab(
+ () => {
+ return batchFetchCollab(workspaceId, params);
+ },
+ params,
+ StrategyType.CACHE_AND_NETWORK,
+ (id: string, doc: YDoc) => {
+ if (!rowsFolder.has(id)) {
+ rowsFolder.set(id, doc);
}
- );
- }
- });
- }
+ }
+ );
+ }
+ });
return {
databaseDoc,
@@ -135,21 +151,6 @@ export class JSDatabaseService implements DatabaseService {
};
}
- async loadDatabaseRows(workspaceId: string, rowIds: string[], rowCallback: (rowId: string, rowDoc: YDoc) => void) {
- try {
- await batchCollabs(
- workspaceId,
- rowIds.map((id) => ({
- object_id: id,
- collab_type: CollabType.DatabaseRow,
- })),
- rowCallback
- );
- } catch (e) {
- console.error(e);
- }
- }
-
async closeDatabase(databaseId: string) {
this.cacheDatabaseRowDocMap.delete(databaseId);
}
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts
index cc3d3dd411..bcf73ae550 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/document.service.ts
@@ -1,8 +1,13 @@
import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';
-import { getCollabStorageWithAPICall, getCurrentWorkspace } from '@/application/services/js-services/storage';
+import { getCollab } from '@/application/services/js-services/cache';
+import { StrategyType } from '@/application/services/js-services/cache/types';
+import { fetchCollab } from '@/application/services/js-services/fetch';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
import { DocumentService } from '@/application/services/services.type';
export class JSDocumentService implements DocumentService {
+ private loaded: Set<string> = new Set();
+
constructor() {
//
}
@@ -14,8 +19,20 @@ export class JSDocumentService implements DocumentService {
throw new Error('Workspace database not found');
}
- const doc = await getCollabStorageWithAPICall(workspace.id, docId, CollabType.Document);
+ const isLoaded = this.loaded.has(docId);
+
+ const doc = await getCollab(
+ () => {
+ return fetchCollab(workspace.id, docId, CollabType.Document);
+ },
+ {
+ collabId: docId,
+ collabType: CollabType.Document,
+ },
+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
+ );
+ if (!isLoaded) this.loaded.add(docId);
const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {
if (origin === CollabOrigin.LocalSync) {
// Send the update to the server
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/fetch.ts b/frontend/appflowy_web_app/src/application/services/js-services/fetch.ts
new file mode 100644
index 0000000000..7ae4dc8902
--- /dev/null
+++ b/frontend/appflowy_web_app/src/application/services/js-services/fetch.ts
@@ -0,0 +1,66 @@
+import { CollabType } from '@/application/collab.type';
+import { APIService } from '@/application/services/js-services/wasm';
+
+const pendingRequests = new Map();
+
+function generateRequestKey<T>(url: string, params: T) {
+ if (!params) return url;
+
+ try {
+ return `${url}_${JSON.stringify(params)}`;
+ } catch (_e) {
+ return `${url}_${params}`;
+ }
+}
+
+// Deduplication fetch requests
+// When multiple requests are made to the same URL with the same params, only one request is made
+// and the result is shared with all the requests
+function fetchWithDeduplication<Req, Res>(url: string, params: Req, fetchFunction: () => Promise<Res>): Promise<Res> {
+ const requestKey = generateRequestKey<Req>(url, params);
+
+ if (pendingRequests.has(requestKey)) {
+ return pendingRequests.get(requestKey);
+ }
+
+ const fetchPromise = fetchFunction().finally(() => {
+ pendingRequests.delete(requestKey);
+ });
+
+ pendingRequests.set(requestKey, fetchPromise);
+ return fetchPromise;
+}
+
+/**
+ * Fetch collab
+ * @param workspaceId
+ * @param id
+ * @param type [CollabType]
+ */
+export function fetchCollab(workspaceId: string, id: string, type: CollabType) {
+ const fetchFunction = () => APIService.getCollab(workspaceId, id, type);
+
+ return fetchWithDeduplication(`fetchCollab_${workspaceId}`, { id, type }, fetchFunction);
+}
+
+/**
+ * Batch fetch collab
+ * Usage:
+ * // load database rows
+ * const rows = await batchFetchCollab(workspaceId, databaseRows.map((row) => ({ collabId: row.id, collabType: CollabType.DatabaseRow })));
+ *
+ * @param workspaceId
+ * @param params [{ collabId: string; collabType: CollabType }]
+ */
+export function batchFetchCollab(workspaceId: string, params: { collabId: string; collabType: CollabType }[]) {
+ const fetchFunction = () =>
+ APIService.batchGetCollab(
+ workspaceId,
+ params.map(({ collabId, collabType }) => ({
+ object_id: collabId,
+ collab_type: collabType,
+ }))
+ );
+
+ return fetchWithDeduplication(`batchFetchCollab_${workspaceId}`, params, fetchFunction);
+}
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts
index c475cfa935..f145480c18 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/folder.service.ts
@@ -1,14 +1,30 @@
import { CollabOrigin, CollabType, YDoc } from '@/application/collab.type';
-import { getCollabStorageWithAPICall } from '@/application/services/js-services/storage';
+import { getCollab } from '@/application/services/js-services/cache';
+import { StrategyType } from '@/application/services/js-services/cache/types';
+import { fetchCollab } from '@/application/services/js-services/fetch';
import { FolderService } from '@/application/services/services.type';
export class JSFolderService implements FolderService {
+ private loaded: Set<string> = new Set();
+
constructor() {
//
}
async openWorkspace(workspaceId: string): Promise<YDoc> {
- const doc = await getCollabStorageWithAPICall(workspaceId, workspaceId, CollabType.Folder);
+ const isLoaded = this.loaded.has(workspaceId);
+ const doc = await getCollab(
+ () => {
+ return fetchCollab(workspaceId, workspaceId, CollabType.Folder);
+ },
+ {
+ collabId: workspaceId,
+ collabType: CollabType.Folder,
+ },
+ isLoaded ? StrategyType.CACHE_FIRST : StrategyType.CACHE_AND_NETWORK
+ );
+
+ if (!isLoaded) this.loaded.add(workspaceId);
const handleUpdate = (update: Uint8Array, origin: CollabOrigin) => {
if (origin === CollabOrigin.LocalSync) {
// Send the update to the server
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/auth.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/auth.ts
similarity index 100%
rename from frontend/appflowy_web_app/src/application/services/js-services/storage/auth.ts
rename to frontend/appflowy_web_app/src/application/services/js-services/session/auth.ts
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/index.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/index.ts
similarity index 73%
rename from frontend/appflowy_web_app/src/application/services/js-services/storage/index.ts
rename to frontend/appflowy_web_app/src/application/services/js-services/session/index.ts
index f0b9cab2d6..c618a85cfd 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/storage/index.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/session/index.ts
@@ -1,4 +1,3 @@
export * from './token';
export * from './user';
-export * from './collab';
export * from './auth';
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/token.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/token.ts
similarity index 100%
rename from frontend/appflowy_web_app/src/application/services/js-services/storage/token.ts
rename to frontend/appflowy_web_app/src/application/services/js-services/session/token.ts
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/user.ts b/frontend/appflowy_web_app/src/application/services/js-services/session/user.ts
similarity index 100%
rename from frontend/appflowy_web_app/src/application/services/js-services/storage/user.ts
rename to frontend/appflowy_web_app/src/application/services/js-services/session/user.ts
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/storage/collab.ts b/frontend/appflowy_web_app/src/application/services/js-services/storage/collab.ts
deleted file mode 100644
index 6b315658f5..0000000000
--- a/frontend/appflowy_web_app/src/application/services/js-services/storage/collab.ts
+++ /dev/null
@@ -1,118 +0,0 @@
-import { CollabType, YDoc, YjsEditorKey, YSharedRoot } from '@/application/collab.type';
-import { getDBName, openCollabDB } from '@/application/services/js-services/db';
-import { APIService } from '@/application/services/js-services/wasm';
-import { applyYDoc } from '@/application/ydoc/apply';
-
-export function fetchCollab(workspaceId: string, id: string, type: CollabType) {
- return APIService.getCollab(workspaceId, id, type);
-}
-
-export function batchFetchCollab(workspaceId: string, params: { object_id: string; collab_type: CollabType }[]) {
- return APIService.batchGetCollab(workspaceId, params);
-}
-
-function collabTypeToDBType(type: CollabType) {
- switch (type) {
- case CollabType.Folder:
- return 'folder';
- case CollabType.Document:
- return 'document';
- case CollabType.Database:
- return 'database';
- case CollabType.WorkspaceDatabase:
- return 'databases';
- case CollabType.DatabaseRow:
- return 'database_row';
- case CollabType.UserAwareness:
- return 'user_awareness';
- default:
- return '';
- }
-}
-
-const collabSharedRootKeyMap = {
- [CollabType.Folder]: YjsEditorKey.folder,
- [CollabType.Document]: YjsEditorKey.document,
- [CollabType.Database]: YjsEditorKey.database,
- [CollabType.WorkspaceDatabase]: YjsEditorKey.workspace_database,
- [CollabType.DatabaseRow]: YjsEditorKey.database_row,
- [CollabType.UserAwareness]: YjsEditorKey.user_awareness,
- [CollabType.Empty]: YjsEditorKey.empty,
-};
-
-export async function getCollabStorage(id: string, type: CollabType) {
- const name = getDBName(id, collabTypeToDBType(type));
-
- const doc = await openCollabDB(name);
- let localExist = false;
- const existData = doc.share.has(YjsEditorKey.data_section);
-
- if (existData) {
- const data = doc.getMap(YjsEditorKey.data_section) as YSharedRoot;
-
- localExist = data.has(collabSharedRootKeyMap[type] as string);
- }
-
- return {
- doc,
- localExist,
- };
-}
-
-export async function getCollabStorageWithAPICall(workspaceId: string, id: string, type: CollabType) {
- const { doc, localExist } = await getCollabStorage(id, type);
- const asyncApply = async () => {
- const res = await fetchCollab(workspaceId, id, type);
-
- applyYDoc(doc, res.state);
- };
-
- // If the document exists locally, apply the state asynchronously,
- // otherwise, apply the state synchronously
- if (localExist) {
- void asyncApply();
- } else {
- await asyncApply();
- }
-
- return doc;
-}
-
-export async function batchCollabs(
- workspaceId: string,
- params: {
- object_id: string;
- collab_type: CollabType;
- }[],
- rowCallback?: (id: string, doc: YDoc) => void
-) {
- console.log('Fetching collab data:', params);
- // Create or get Y.Doc from local storage
- for (const item of params) {
- const { object_id, collab_type } = item;
-
- const { doc, localExist } = await getCollabStorage(object_id, collab_type);
-
- if (rowCallback && localExist) {
- rowCallback(object_id, doc);
- }
- }
-
- const res = await batchFetchCollab(workspaceId, params);
-
- console.log('Fetched collab data:', res);
- for (const id of Object.keys(res)) {
- const type = params.find((param) => param.object_id === id)?.collab_type;
- const data = res[id];
-
- if (type === undefined || !data) {
- continue;
- }
-
- const { doc } = await getCollabStorage(id, type);
-
- applyYDoc(doc, new Uint8Array(data));
-
- rowCallback?.(id, doc);
- }
-}
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts b/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts
index c4853f850d..ce912bd50f 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/user.service.ts
@@ -8,7 +8,7 @@ import {
invalidToken,
setSignInUser,
setUserWorkspace,
-} from '@/application/services/js-services/storage';
+} from 'src/application/services/js-services/session';
import { asyncDataDecorator } from '@/application/services/js-services/decorator';
async function getUser() {
diff --git a/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts b/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts
index 6b50ca27e5..cd09cb74d1 100644
--- a/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts
+++ b/frontend/appflowy_web_app/src/application/services/js-services/wasm/client_api.ts
@@ -2,7 +2,7 @@ import { CollabType } from '@/application/collab.type';
import { ClientAPI } from '@appflowyinc/client-api-wasm';
import { UserProfile, UserWorkspace } from '@/application/user.type';
import { AFCloudConfig } from '@/application/services/services.type';
-import { invalidToken, readTokenStr, writeToken } from '@/application/services/js-services/storage';
+import { invalidToken, readTokenStr, writeToken } from 'src/application/services/js-services/session';
let client: ClientAPI;
diff --git a/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx b/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx
index 76d0f7575c..a7ef1d2684 100644
--- a/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx
+++ b/frontend/appflowy_web_app/src/components/_shared/not-found/RecordNotFound.tsx
@@ -1,4 +1,4 @@
-import { getCurrentWorkspace } from '@/application/services/js-services/storage';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
import { Button, Dialog, DialogActions, DialogContent, DialogContentText, DialogTitle } from '@mui/material';
import React from 'react';
import { useNavigate } from 'react-router-dom';
diff --git a/frontend/appflowy_web_app/src/components/database/Database.tsx b/frontend/appflowy_web_app/src/components/database/Database.tsx
index 6c7c3a4868..0c590462f9 100644
--- a/frontend/appflowy_web_app/src/components/database/Database.tsx
+++ b/frontend/appflowy_web_app/src/components/database/Database.tsx
@@ -12,7 +12,6 @@ export const Database = memo(
viewId: string;
onNavigateToView: (viewId: string) => void;
}) => {
- console.log('Database', viewId, iidIndex);
return (
<div className='appflowy-database relative flex w-full flex-1 select-text flex-col overflow-y-hidden'>
<DatabaseViews iidIndex={iidIndex} onChangeView={onNavigateToView} viewId={viewId} />
diff --git a/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx b/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx
index a563557ab7..1e6bbb151a 100644
--- a/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx
+++ b/frontend/appflowy_web_app/src/components/editor/components/blocks/database/DatabaseBlock.tsx
@@ -1,6 +1,6 @@
import { ReactComponent as ExpandMoreIcon } from '$icons/16x/full_view.svg';
import { useNavigateToView } from '@/application/folder-yjs';
-import { getCurrentWorkspace } from '@/application/services/js-services/storage';
+import { getCurrentWorkspace } from 'src/application/services/js-services/session';
import { IdProvider } from '@/components/_shared/context-provider/IdProvider';
import { Database } from '@/components/database';
import { useGetDatabaseId, useLoadDatabase } from '@/components/database/Database.hooks';
diff --git a/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx b/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx
index 1f58950e58..7d857ac893 100644
--- a/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx
+++ b/frontend/appflowy_web_app/src/components/layout/breadcrumb/Item.tsx
@@ -9,19 +9,19 @@ import { ReactComponent as CalendarSvg } from '$icons/16x/date.svg';
const renderCrumbIcon = (icon: string) => {
if (Number(icon) === ViewLayout.Grid) {
- return <GridSvg />;
+ return <GridSvg className={'h-4 w-4'} />;
}
if (Number(icon) === ViewLayout.Board) {
- return <BoardSvg />;
+ return <BoardSvg className={'h-4 w-4'} />;
}
if (Number(icon) === ViewLayout.Calendar) {
- return <CalendarSvg />;
+ return <CalendarSvg className={'h-4 w-4'} />;
}
if (Number(icon) === ViewLayout.Document) {
- return <DocumentSvg />;
+ return <DocumentSvg className={'h-4 w-4'} />;
}
return icon;
@@ -41,7 +41,7 @@ function Item({ crumb, disableClick = false }: { crumb: Crumb; disableClick?: bo
onNavigateToView?.(viewId);
}}
>
- <span className={'h-4 w-4'}>{renderCrumbIcon(icon)}</span>
+ {renderCrumbIcon(icon)}
<span
className={!disableClick ? 'max-w-[250px] truncate hover:text-fill-default hover:underline' : 'flex-1 truncate'}
>