There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat(nbstore): add cloud implementation (#8810)
forehalo2f80b4f822f4
diff --git a/packages/backend/server/migrations/20241023065619_blobs/migration.sql b/packages/backend/server/migrations/20241023065619_blobs/migration.sqlnew file mode 100644index 0000000000..22fac2f058--- /dev/null+++ b/packages/backend/server/migrations/20241023065619_blobs/migration.sql@@ -0,0 +1,14 @@+-- CreateTable+CREATE TABLE "blobs" (+ "workspace_id" VARCHAR NOT NULL,+ "key" VARCHAR NOT NULL,+ "size" INTEGER NOT NULL,+ "mime" VARCHAR NOT NULL,+ "created_at" TIMESTAMPTZ(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,+ "deleted_at" TIMESTAMPTZ(3),++ CONSTRAINT "blobs_pkey" PRIMARY KEY ("workspace_id","key")+);++-- AddForeignKey+ALTER TABLE "blobs" ADD CONSTRAINT "blobs_workspace_id_fkey" FOREIGN KEY ("workspace_id") REFERENCES "workspaces"("id") ON DELETE CASCADE ON UPDATE CASCADE;diff --git a/packages/backend/server/schema.prisma b/packages/backend/server/schema.prismaindex 9f810ceef4..ce5315b46e 100644--- a/packages/backend/server/schema.prisma+++ b/packages/backend/server/schema.prisma@@ -106,6 +106,7 @@ model Workspace {permissions WorkspaceUserPermission[]pagePermissions WorkspacePageUserPermission[]features WorkspaceFeature[]+ blobs Blob[]@@map("workspaces")}@@ -568,3 +569,19 @@ model Invoice {@@index([targetId])@@map("invoices")}+// Blob table only exists for fast non-data queries.+// like, total size of blobs in a workspace, or blob list for sync service.+// it should only be a map of metadata of blobs stored anywhere else+model Blob {+ workspaceId String @map("workspace_id") @db.VarChar+ key String @db.VarChar+ size Int @db.Integer+ mime String @db.VarChar+ createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz(3)+ deletedAt DateTime? @map("deleted_at") @db.Timestamptz(3)++ workspace Workspace @relation(fields: [workspaceId], references: [id], onDelete: Cascade)++ @@id([workspaceId, key])+ @@map("blobs")+}diff --git a/packages/backend/server/src/core/doc/storage/doc.ts b/packages/backend/server/src/core/doc/storage/doc.tsindex 6010855484..4bcbf4067b 100644--- a/packages/backend/server/src/core/doc/storage/doc.ts+++ b/packages/backend/server/src/core/doc/storage/doc.ts@@ -1,8 +1,10 @@import {applyUpdate,+ diffUpdate,Doc,encodeStateAsUpdate,encodeStateVector,+ encodeStateVectorFromUpdate,mergeUpdates,UndoManager,} from 'yjs';@@ -19,6 +21,12 @@ export interface DocRecord {editor?: string;}+export interface DocDiff {+ missing: Uint8Array;+ state: Uint8Array;+ timestamp: number;+}+export interface DocUpdate {bin: Uint8Array;timestamp: number;@@ -96,6 +104,27 @@ export abstract class DocStorageAdapter extends Connection {return snapshot;}+ async getDocDiff(+ spaceId: string,+ docId: string,+ stateVector?: Uint8Array+ ): Promise<DocDiff | null> {+ const doc = await this.getDoc(spaceId, docId);++ if (!doc) {+ return null;+ }++ const missing = stateVector ? diffUpdate(doc.bin, stateVector) : doc.bin;+ const state = encodeStateVectorFromUpdate(doc.bin);++ return {+ missing,+ state,+ timestamp: doc.timestamp,+ };+ }+abstract pushDocUpdates(spaceId: string,docId: string,diff --git a/packages/backend/server/src/core/doc/storage/index.ts b/packages/backend/server/src/core/doc/storage/index.tsindex 6ba0e23dd1..7521fb6cca 100644--- a/packages/backend/server/src/core/doc/storage/index.ts+++ b/packages/backend/server/src/core/doc/storage/index.ts@@ -1,4 +1,6 @@-// TODO(@forehalo): share with frontend+// This is a totally copy of definitions in [@affine/space-store]+// because currently importing cross workspace package from [@affine/server] is not yet supported+// should be kept updated with the original definitions in [@affine/space-store]import type { BlobStorageAdapter } from './blob';import { Connection } from './connection';import type { DocStorageAdapter } from './doc';diff --git a/packages/backend/server/src/core/quota/resolver.ts b/packages/backend/server/src/core/quota/resolver.tsindex c8afaa2237..309f43833a 100644--- a/packages/backend/server/src/core/quota/resolver.ts+++ b/packages/backend/server/src/core/quota/resolver.ts@@ -11,6 +11,7 @@ import { CurrentUser } from '../auth/session';import { EarlyAccessType } from '../features';import { UserType } from '../user';import { QuotaService } from './service';+import { QuotaManagementService } from './storage';registerEnumType(EarlyAccessType, {name: 'EarlyAccessType',@@ -55,9 +56,18 @@ class UserQuotaType {humanReadable!: UserQuotaHumanReadableType;}+@ObjectType('UserQuotaUsage')+class UserQuotaUsageType {+ @Field(() => SafeIntResolver, { name: 'storageQuota' })+ storageQuota!: number;+}+@Resolver(() => UserType)export class QuotaManagementResolver {- constructor(private readonly quota: QuotaService) {}+ constructor(+ private readonly quota: QuotaService,+ private readonly management: QuotaManagementService+ ) {}@ResolveField(() => UserQuotaType, { name: 'quota', nullable: true })async getQuota(@CurrentUser() me: UserType) {@@ -65,4 +75,15 @@ export class QuotaManagementResolver {return quota.feature;}++ @ResolveField(() => UserQuotaUsageType, { name: 'quotaUsage' })+ async getQuotaUsage(+ @CurrentUser() me: UserType+ ): Promise<UserQuotaUsageType> {+ const usage = await this.management.getUserStorageUsage(me.id);++ return {+ storageQuota: usage,+ };+ }}diff --git a/packages/backend/server/src/core/quota/storage.ts b/packages/backend/server/src/core/quota/storage.tsindex 8476a48973..a74942eeec 100644--- a/packages/backend/server/src/core/quota/storage.ts+++ b/packages/backend/server/src/core/quota/storage.ts@@ -77,7 +77,7 @@ export class QuotaManagementService {return this.quota.hasWorkspaceQuota(workspaceId, QuotaType.TeamPlanV1);}- async getUserUsage(userId: string) {+ async getUserStorageUsage(userId: string) {const workspaces = await this.permissions.getOwnedWorkspaces(userId);const sizes = await Promise.allSettled(@@ -125,7 +125,7 @@ export class QuotaManagementService {async getQuotaCalculator(userId: string) {const quota = await this.getUserQuota(userId);const { storageQuota, businessBlobLimit } = quota;- const usedSize = await this.getUserUsage(userId);+ const usedSize = await this.getUserStorageUsage(userId);return this.generateQuotaCalculator(storageQuota,@@ -183,7 +183,7 @@ export class QuotaManagementService {humanReadable,} = await this.getWorkspaceQuota(owner.id, workspaceId);// get all workspaces size of owner used- const usedSize = await this.getUserUsage(owner.id);+ const usedSize = await this.getUserStorageUsage(owner.id);// relax restrictions if workspace has unlimited feature// todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quotaconst unlimited = await this.feature.hasWorkspaceFeature(diff --git a/packages/backend/server/src/core/storage/wrappers/blob.ts b/packages/backend/server/src/core/storage/wrappers/blob.tsindex 9aa991ab4b..38b63df950 100644--- a/packages/backend/server/src/core/storage/wrappers/blob.ts+++ b/packages/backend/server/src/core/storage/wrappers/blob.ts@@ -1,33 +1,42 @@-import { Injectable } from '@nestjs/common';+import { Injectable, Logger } from '@nestjs/common';+import { PrismaClient } from '@prisma/client';import {- type BlobInputType,- Cache,+ autoMetadata,Config,EventEmitter,type EventPayload,- type ListObjectsMetadata,+ type GetObjectMetadata,+ ListObjectsMetadata,OnEvent,+ PutObjectMetadata,type StorageProvider,StorageProviderFactory,} from '../../../fundamentals';@Injectable()export class WorkspaceBlobStorage {+ private readonly logger = new Logger(WorkspaceBlobStorage.name);public readonly provider: StorageProvider;constructor(private readonly config: Config,private readonly event: EventEmitter,private readonly storageFactory: StorageProviderFactory,- private readonly cache: Cache+ private readonly db: PrismaClient) {this.provider = this.storageFactory.create(this.config.storages.blob);}- async put(workspaceId: string, key: string, blob: BlobInputType) {- await this.provider.put(`${workspaceId}/${key}`, blob);- await this.cache.delete(`blob-list:${workspaceId}`);+ async put(workspaceId: string, key: string, blob: Buffer) {+ const meta: PutObjectMetadata = autoMetadata(blob);++ await this.provider.put(`${workspaceId}/${key}`, blob, meta);+ this.trySyncBlobMeta(workspaceId, key, {+ contentType: meta.contentType ?? 'application/octet-stream',+ contentLength: blob.length,+ lastModified: new Date(),+ });}async get(workspaceId: string, key: string) {@@ -35,41 +44,141 @@ export class WorkspaceBlobStorage {}async list(workspaceId: string) {- const cachedList = await this.cache.list<ListObjectsMetadata>(- `blob-list:${workspaceId}`,- 0,- -1- );-- if (cachedList.length > 0) {- return cachedList;+ const blobsInDb = await this.db.blob.findMany({+ where: {+ workspaceId,+ deletedAt: null,+ },+ });++ if (blobsInDb.length > 0) {+ return blobsInDb;}const blobs = await this.provider.list(workspaceId + '/');-- blobs.forEach(item => {- // trim workspace prefix- item.key = item.key.slice(workspaceId.length + 1);+ blobs.forEach(blob => {+ blob.key = blob.key.slice(workspaceId.length + 1);});- await this.cache.pushBack(`blob-list:${workspaceId}`, ...blobs);+ this.trySyncBlobsMeta(workspaceId, blobs);- return blobs;+ return blobs.map(blob => ({+ key: blob.key,+ size: blob.contentLength,+ createdAt: blob.lastModified,+ mime: 'application/octet-stream',+ }));}- /**- * we won't really delete the blobs until the doc blobs manager is implemented sounded- */- async delete(_workspaceId: string, _key: string) {- // return this.provider.delete(`${workspaceId}/${key}`);+ async delete(workspaceId: string, key: string, permanently = false) {+ if (permanently) {+ await this.provider.delete(`${workspaceId}/${key}`);+ await this.db.blob.deleteMany({+ where: {+ workspaceId,+ key,+ },+ });+ } else {+ await this.db.blob.update({+ where: {+ workspaceId_key: {+ workspaceId,+ key,+ },+ },+ data: {+ deletedAt: new Date(),+ },+ });+ }+ }++ async release(workspaceId: string) {+ const deletedBlobs = await this.db.blob.findMany({+ where: {+ workspaceId,+ deletedAt: {+ not: null,+ },+ },+ });++ deletedBlobs.forEach(blob => {+ this.event.emit('workspace.blob.deleted', {+ workspaceId: workspaceId,+ key: blob.key,+ });+ });}async totalSize(workspaceId: string) {const blobs = await this.list(workspaceId);- // how could we ignore the ones get soft-deleted?return blobs.reduce((acc, item) => acc + item.size, 0);}+ private trySyncBlobsMeta(workspaceId: string, blobs: ListObjectsMetadata[]) {+ for (const blob of blobs) {+ this.trySyncBlobMeta(workspaceId, blob.key);+ }+ }++ private trySyncBlobMeta(+ workspaceId: string,+ key: string,+ meta?: GetObjectMetadata+ ) {+ setImmediate(() => {+ this.syncBlobMeta(workspaceId, key, meta).catch(() => {+ /* never throw */+ });+ });+ }++ private async syncBlobMeta(+ workspaceId: string,+ key: string,+ meta?: GetObjectMetadata+ ) {+ try {+ if (!meta) {+ const blob = await this.get(workspaceId, key);+ meta = blob.metadata;+ }++ if (meta) {+ await this.db.blob.upsert({+ where: {+ workspaceId_key: {+ workspaceId,+ key,+ },+ },+ update: {+ mime: meta.contentType,+ size: meta.contentLength,+ },+ create: {+ workspaceId,+ key,+ mime: meta.contentType,+ size: meta.contentLength,+ },+ });+ } else {+ await this.db.blob.deleteMany({+ where: {+ workspaceId,+ key,+ },+ });+ }+ } catch (e) {+ // never throw+ this.logger.error('failed to sync blob meta to DB', e);+ }+ }+@OnEvent('workspace.deleted')async onWorkspaceDeleted(workspaceId: EventPayload<'workspace.deleted'>) {const blobs = await this.list(workspaceId);@@ -78,7 +187,7 @@ export class WorkspaceBlobStorage {blobs.forEach(blob => {this.event.emit('workspace.blob.deleted', {workspaceId: workspaceId,- name: blob.key,+ key: blob.key,});});}@@ -86,8 +195,8 @@ export class WorkspaceBlobStorage {@OnEvent('workspace.blob.deleted')async onDeleteWorkspaceBlob({workspaceId,- name,+ key,}: EventPayload<'workspace.blob.deleted'>) {- await this.delete(workspaceId, name);+ await this.delete(workspaceId, key, true);}}diff --git a/packages/backend/server/src/core/sync/gateway.ts b/packages/backend/server/src/core/sync/gateway.tsindex d5686c658f..d1c1c5ff77 100644--- a/packages/backend/server/src/core/sync/gateway.ts+++ b/packages/backend/server/src/core/sync/gateway.ts@@ -8,7 +8,6 @@ import {WebSocketGateway,} from '@nestjs/websockets';import { Socket } from 'socket.io';-import { diffUpdate, encodeStateVectorFromUpdate } from 'yjs';import {AlreadyInSpace,@@ -83,6 +82,9 @@ interface LeaveSpaceAwarenessMessage {docId: string;}+/**+ * @deprecated+ */interface PushDocUpdatesMessage {spaceType: SpaceType;spaceId: string;@@ -90,6 +92,13 @@ interface PushDocUpdatesMessage {updates: string[];}+interface PushDocUpdateMessage {+ spaceType: SpaceType;+ spaceId: string;+ docId: string;+ update: string;+}+interface LoadDocMessage {spaceType: SpaceType;spaceId: string;@@ -97,6 +106,12 @@ interface LoadDocMessage {stateVector?: string;}+interface DeleteDocMessage {+ spaceType: SpaceType;+ spaceId: string;+ docId: string;+}+interface LoadDocTimestampsMessage {spaceType: SpaceType;spaceId: string;@@ -114,6 +129,7 @@ interface UpdateAwarenessMessage {docId: string;awarenessUpdate: string;}+@WebSocketGateway()export class SpaceSyncGatewayimplements OnGatewayConnection, OnGatewayDisconnect@@ -182,26 +198,6 @@ export class SpaceSyncGateway}}- async joinWorkspace(- client: Socket,- room: `${string}:${'sync' | 'awareness'}`- ) {- await client.join(room);- }-- async leaveWorkspace(- client: Socket,- room: `${string}:${'sync' | 'awareness'}`- ) {- await client.leave(room);- }-- assertInWorkspace(client: Socket, room: `${string}:${'sync' | 'awareness'}`) {- if (!client.rooms.has(room)) {- throw new NotInSpace({ spaceId: room.split(':')[0] });- }- }-// v3@SubscribeMessage('space:join')async onJoinSpace(@@ -233,36 +229,42 @@ export class SpaceSyncGateway@MessageBody(){ spaceType, spaceId, docId, stateVector }: LoadDocMessage): Promise<- EventResponse<{ missing: string; state?: string; timestamp: number }>+ EventResponse<{ missing: string; state: string; timestamp: number }>> {const adapter = this.selectAdapter(client, spaceType);adapter.assertIn(spaceId);- const doc = await adapter.get(spaceId, docId);+ const doc = await adapter.diff(+ spaceId,+ docId,+ stateVector ? Buffer.from(stateVector, 'base64') : undefined+ );if (!doc) {throw new DocNotFound({ spaceId, docId });}- const missing = Buffer.from(- stateVector- ? diffUpdate(doc.bin, Buffer.from(stateVector, 'base64'))- : doc.bin- ).toString('base64');-- const state = Buffer.from(encodeStateVectorFromUpdate(doc.bin)).toString(- 'base64'- );-return {data: {- missing,- state,+ missing: Buffer.from(doc.missing).toString('base64'),+ state: Buffer.from(doc.state).toString('base64'),timestamp: doc.timestamp,},};}+ @SubscribeMessage('space:delete-doc')+ async onDeleteSpaceDoc(+ @ConnectedSocket() client: Socket,+ @MessageBody() { spaceType, spaceId, docId }: DeleteDocMessage+ ) {+ const adapter = this.selectAdapter(client, spaceType);+ await adapter.delete(spaceId, docId);+ }++ /**+ * @deprecated use [space:push-doc-update] instead, client should always merge updates on their own+ */@SubscribeMessage('space:push-doc-updates')async onReceiveDocUpdates(@ConnectedSocket() client: Socket,@@ -307,6 +309,51 @@ export class SpaceSyncGateway};}+ @SubscribeMessage('space:push-doc-update')+ async onReceiveDocUpdate(+ @ConnectedSocket() client: Socket,+ @CurrentUser() user: CurrentUser,+ @MessageBody()+ message: PushDocUpdateMessage+ ): Promise<EventResponse<{ accepted: true; timestamp?: number }>> {+ const { spaceType, spaceId, docId, update } = message;+ const adapter = this.selectAdapter(client, spaceType);++ // TODO(@forehalo): we might need to check write permission before push updates+ const timestamp = await adapter.push(+ spaceId,+ docId,+ [Buffer.from(update, 'base64')],+ user.id+ );++ // TODO(@forehalo): separate different version of clients into different rooms,+ // so the clients won't receive useless updates events+ client.to(adapter.room(spaceId)).emit('space:broadcast-doc-updates', {+ spaceType,+ spaceId,+ docId,+ updates: [update],+ timestamp,+ });++ client.to(adapter.room(spaceId)).emit('space:broadcast-doc-update', {+ spaceType,+ spaceId,+ docId,+ update,+ timestamp,+ editor: user.id,+ });++ return {+ data: {+ accepted: true,+ timestamp,+ },+ };+ }+@SubscribeMessage('space:load-doc-timestamps')async onLoadDocTimestamps(@ConnectedSocket() client: Socket,@@ -600,9 +647,14 @@ abstract class SyncSocketAdapter {return this.storage.pushDocUpdates(spaceId, docId, updates, editorId);}- get(spaceId: string, docId: string) {+ diff(spaceId: string, docId: string, stateVector?: Uint8Array) {+ this.assertIn(spaceId);+ return this.storage.getDocDiff(spaceId, docId, stateVector);+ }++ delete(spaceId: string, docId: string) {this.assertIn(spaceId);- return this.storage.getDoc(spaceId, docId);+ return this.storage.deleteDoc(spaceId, docId);}getTimestamps(spaceId: string, timestamp?: number) {@@ -630,9 +682,9 @@ class WorkspaceSyncAdapter extends SyncSocketAdapter {return super.push(spaceId, id.guid, updates, editorId);}- override get(spaceId: string, docId: string) {+ override diff(spaceId: string, docId: string, stateVector?: Uint8Array) {const id = new DocID(docId, spaceId);- return this.storage.getDoc(spaceId, id.guid);+ return this.storage.getDocDiff(spaceId, id.guid, stateVector);}async assertAccessible(diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.tsindex da61143a99..448d23741c 100644--- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts+++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts@@ -1,29 +1,40 @@import { Logger, UseGuards } from '@nestjs/common';import {Args,+ Field,Int,Mutation,+ ObjectType,Parent,Query,ResolveField,Resolver,} from '@nestjs/graphql';-import { SafeIntResolver } from 'graphql-scalars';import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';import type { FileUpload } from '../../../fundamentals';-import {- BlobQuotaExceeded,- CloudThrottlerGuard,- MakeCache,- PreventCache,-} from '../../../fundamentals';+import { BlobQuotaExceeded, CloudThrottlerGuard } from '../../../fundamentals';import { CurrentUser } from '../../auth';import { Permission, PermissionService } from '../../permission';import { QuotaManagementService } from '../../quota';import { WorkspaceBlobStorage } from '../../storage';import { WorkspaceBlobSizes, WorkspaceType } from '../types';+@ObjectType()+class ListedBlob {+ @Field()+ key!: string;++ @Field()+ mime!: string;++ @Field()+ size!: number;++ @Field()+ createdAt!: string;+}+@UseGuards(CloudThrottlerGuard)@Resolver(() => WorkspaceType)export class WorkspaceBlobResolver {@@ -34,7 +45,7 @@ export class WorkspaceBlobResolver {private readonly storage: WorkspaceBlobStorage) {}- @ResolveField(() => [String], {+ @ResolveField(() => [ListedBlob], {description: 'List blobs of workspace',complexity: 2,})@@ -44,9 +55,7 @@ export class WorkspaceBlobResolver {) {await this.permissions.checkWorkspace(workspace.id, user.id);- return this.storage- .list(workspace.id)- .then(list => list.map(item => item.key));+ return this.storage.list(workspace.id);}@ResolveField(() => Int, {@@ -64,7 +73,6 @@ export class WorkspaceBlobResolver {description: 'List blobs of workspace',deprecationReason: 'use `workspace.blobs` instead',})- @MakeCache(['blobs'], ['workspaceId'])async listBlobs(@CurrentUser() user: CurrentUser,@Args('workspaceId') workspaceId: string@@ -76,42 +84,15 @@ export class WorkspaceBlobResolver {.then(list => list.map(item => item.key));}- /**- * @deprecated use `user.storageUsage` instead- */@Query(() => WorkspaceBlobSizes, {- deprecationReason: 'use `user.storageUsage` instead',+ deprecationReason: 'use `user.quotaUsage` instead',})async collectAllBlobSizes(@CurrentUser() user: CurrentUser) {- const size = await this.quota.getUserUsage(user.id);+ const size = await this.quota.getUserStorageUsage(user.id);return { size };}- /**- * @deprecated mutation `setBlob` will check blob limit & quota usage- */- @Query(() => WorkspaceBlobSizes, {- deprecationReason: 'no more needed',- })- async checkBlobSize(- @CurrentUser() user: CurrentUser,- @Args('workspaceId') workspaceId: string,- @Args('size', { type: () => SafeIntResolver }) blobSize: number- ) {- const canWrite = await this.permissions.tryCheckWorkspace(- workspaceId,- user.id,- Permission.Write- );- if (canWrite) {- const size = await this.quota.checkBlobQuota(workspaceId, blobSize);- return { size };- }- return false;- }-@Mutation(() => String)- @PreventCache(['blobs'], ['workspaceId'])async setBlob(@CurrentUser() user: CurrentUser,@Args('workspaceId') workspaceId: string,@@ -160,11 +141,35 @@ export class WorkspaceBlobResolver {}@Mutation(() => Boolean)- @PreventCache(['blobs'], ['workspaceId'])async deleteBlob(@CurrentUser() user: CurrentUser,@Args('workspaceId') workspaceId: string,- @Args('hash') name: string+ @Args('hash', {+ type: () => String,+ deprecationReason: 'use parameter [key]',+ nullable: true,+ })+ hash?: string,+ @Args('key', { type: () => String, nullable: true }) key?: string,+ @Args('permanently', { type: () => Boolean, defaultValue: false })+ permanently = false+ ) {+ key = key ?? hash;+ if (!key) {+ return false;+ }++ await this.permissions.checkWorkspace(workspaceId, user.id);++ await this.storage.delete(workspaceId, key, permanently);++ return true;+ }++ @Mutation(() => Boolean)+ async releaseDeletedBlobs(+ @CurrentUser() user: CurrentUser,+ @Args('workspaceId') workspaceId: string) {await this.permissions.checkWorkspace(workspaceId,@@ -172,7 +177,7 @@ export class WorkspaceBlobResolver {Permission.Write);- await this.storage.delete(workspaceId, name);+ await this.storage.release(workspaceId);return true;}diff --git a/packages/backend/server/src/fundamentals/event/def.ts b/packages/backend/server/src/fundamentals/event/def.tsindex 33db14b7a2..c9cb423813 100644--- a/packages/backend/server/src/fundamentals/event/def.ts+++ b/packages/backend/server/src/fundamentals/event/def.ts@@ -7,7 +7,7 @@ export interface WorkspaceEvents {blob: {deleted: Payload<{workspaceId: Workspace['id'];- name: string;+ key: string;}>;};}diff --git a/packages/backend/server/src/fundamentals/storage/providers/fs.ts b/packages/backend/server/src/fundamentals/storage/providers/fs.tsindex 282396c98d..41e17f3e66 100644--- a/packages/backend/server/src/fundamentals/storage/providers/fs.ts+++ b/packages/backend/server/src/fundamentals/storage/providers/fs.ts@@ -119,7 +119,7 @@ export class FsStorageProvider implements StorageProvider {results.push({key: res,lastModified: stat.mtime,- size: stat.size,+ contentLength: stat.size,});}}@@ -216,7 +216,7 @@ export class FsStorageProvider implements StorageProvider {raw: PutObjectMetadata) {try {- const metadata = await autoMetadata(blob, raw);+ const metadata = autoMetadata(blob, raw);if (raw.checksumCRC32 && metadata.checksumCRC32 !== raw.checksumCRC32) {throw new Error(@@ -224,6 +224,12 @@ export class FsStorageProvider implements StorageProvider {);}+ if (raw.contentLength && metadata.contentLength !== raw.contentLength) {+ throw new Error(+ 'The content length of the uploaded file is not matched with the one you provide, the file may be corrupted and the uploading will not be processed.'+ );+ }+writeFileSync(this.join(`${key}.metadata.json`),JSON.stringify({diff --git a/packages/backend/server/src/fundamentals/storage/providers/provider.ts b/packages/backend/server/src/fundamentals/storage/providers/provider.tsindex 46f8ef688d..5b2bad24b2 100644--- a/packages/backend/server/src/fundamentals/storage/providers/provider.ts+++ b/packages/backend/server/src/fundamentals/storage/providers/provider.ts@@ -21,7 +21,7 @@ export interface PutObjectMetadata {export interface ListObjectsMetadata {key: string;lastModified: Date;- size: number;+ contentLength: number;}export type BlobInputType = Buffer | Readable | string;diff --git a/packages/backend/server/src/fundamentals/storage/providers/utils.ts b/packages/backend/server/src/fundamentals/storage/providers/utils.tsindex a0eab7d8c5..4c7e9323f2 100644--- a/packages/backend/server/src/fundamentals/storage/providers/utils.ts+++ b/packages/backend/server/src/fundamentals/storage/providers/utils.ts@@ -14,19 +14,19 @@ export async function toBuffer(input: BlobInputType): Promise<Buffer> {: Buffer.from(input);}-export async function autoMetadata(+export function autoMetadata(blob: Buffer,- raw: PutObjectMetadata-): Promise<PutObjectMetadata> {+ raw: PutObjectMetadata = {}+): PutObjectMetadata {const metadata = {...raw,};- try {- // length- if (!metadata.contentLength) {- metadata.contentLength = blob.length;- }+ if (!metadata.contentLength) {+ metadata.contentLength = blob.byteLength;+ }++ try {// checksumif (!metadata.checksumCRC32) {metadata.checksumCRC32 = crc32(blob).toString(16);@@ -34,15 +34,11 @@ export async function autoMetadata(// mime typeif (!metadata.contentType) {- try {- metadata.contentType = getMime(blob);- } catch {- // ignore- }+ metadata.contentType = getMime(blob);}-- return metadata;} catch {- return metadata;+ // noop}++ return metadata;}diff --git a/packages/backend/server/src/plugins/storage/providers/s3.ts b/packages/backend/server/src/plugins/storage/providers/s3.tsindex 3a242704db..451bd5f6af 100644--- a/packages/backend/server/src/plugins/storage/providers/s3.ts+++ b/packages/backend/server/src/plugins/storage/providers/s3.ts@@ -50,7 +50,7 @@ export class S3StorageProvider implements StorageProvider {): Promise<void> {const blob = await toBuffer(body);- metadata = await autoMetadata(blob, metadata);+ metadata = autoMetadata(blob, metadata);try {await this.client.send(@@ -140,7 +140,7 @@ export class S3StorageProvider implements StorageProvider {listResult.Contents.map(r => ({key: r.Key!,lastModified: r.LastModified!,- size: r.Size!,+ contentLength: r.Size!,})));}diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gqlindex c138ef63f3..c463efc2b3 100644--- a/packages/backend/server/src/schema.gql+++ b/packages/backend/server/src/schema.gql@@ -449,6 +449,13 @@ input ListUserInput {skip: Int = 0}+type ListedBlob {+ createdAt: String!+ key: String!+ mime: String!+ size: Int!+}+input ManageUserInput {"""User email"""email: String@@ -496,7 +503,7 @@ type Mutation {"""Create a new workspace"""createWorkspace(init: Upload): WorkspaceType!deleteAccount: DeleteAccount!- deleteBlob(hash: String!, workspaceId: String!): Boolean!+ deleteBlob(hash: String @deprecated(reason: "use parameter [key]"), key: String, permanently: Boolean! = false, workspaceId: String!): Boolean!"""Delete a user account"""deleteUser(id: String!): DeleteAccount!@@ -511,6 +518,7 @@ type Mutation {leaveWorkspace(sendLeaveMail: Boolean, workspaceId: String!, workspaceName: String!): Boolean!publishPage(mode: PublicPageMode = Page, pageId: String!, workspaceId: String!): WorkspacePage!recoverDoc(guid: String!, timestamp: DateTime!, workspaceId: String!): DateTime!+ releaseDeletedBlobs(workspaceId: String!): Boolean!"""Remove user avatar"""removeAvatar: RemoveAvatar!@@ -584,8 +592,7 @@ enum PublicPageMode {}type Query {- checkBlobSize(size: SafeInt!, workspaceId: String!): WorkspaceBlobSizes! @deprecated(reason: "no more needed")- collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.storageUsage` instead")+ collectAllBlobSizes: WorkspaceBlobSizes! @deprecated(reason: "use `user.quotaUsage` instead")"""Get current user"""currentUser: UserType@@ -885,6 +892,10 @@ type UserQuotaHumanReadable {storageQuota: String!}+type UserQuotaUsage {+ storageQuota: SafeInt!+}+type UserType {"""User avatar url"""avatarUrl: String@@ -913,6 +924,7 @@ type UserType {"""User name"""name: String!quota: UserQuota+ quotaUsage: UserQuotaUsage!subscriptions: [SubscriptionType!]!token: tokenType! @deprecated(reason: "use [/api/auth/sign-in?native=true] instead")}@@ -962,7 +974,7 @@ type WorkspaceType {availableFeatures: [FeatureType!]!"""List blobs of workspace"""- blobs: [String!]!+ blobs: [ListedBlob!]!"""Blobs size of workspace"""blobsSize: Int!diff --git a/packages/backend/server/tests/utils/blobs.ts b/packages/backend/server/tests/utils/blobs.tsindex 514fbeee46..f6832c41c4 100644--- a/packages/backend/server/tests/utils/blobs.ts+++ b/packages/backend/server/tests/utils/blobs.ts@@ -54,35 +54,16 @@ export async function collectAllBlobSizes(.send({query: `query {- collectAllBlobSizes {- size+ currentUser {+ quotaUsage {+ storageQuota+ }}}`,}).expect(200);- return res.body.data.collectAllBlobSizes.size;-}--export async function checkBlobSize(- app: INestApplication,- token: string,- workspaceId: string,- size: number-): Promise<number> {- const res = await request(app.getHttpServer())- .post(gql)- .auth(token, { type: 'bearer' })- .send({- query: `query checkBlobSize($workspaceId: String!, $size: SafeInt!) {- checkBlobSize(workspaceId: $workspaceId, size: $size) {- size- }- }`,- variables: { workspaceId, size },- })- .expect(200);- return res.body.data.checkBlobSize.size;+ return res.body.data.currentUser.quotaUsage.storageQuota;}export async function setBlob(diff --git a/packages/backend/server/tests/workspace-blobs.e2e.ts b/packages/backend/server/tests/workspace/blobs.e2e.tssimilarity index 76%rename from packages/backend/server/tests/workspace-blobs.e2e.tsrename to packages/backend/server/tests/workspace/blobs.e2e.tsindex 3e81d5fbf7..08dc6947fc 100644--- a/packages/backend/server/tests/workspace-blobs.e2e.ts+++ b/packages/backend/server/tests/workspace/blobs.e2e.ts@@ -2,11 +2,10 @@ import type { INestApplication } from '@nestjs/common';import test from 'ava';import request from 'supertest';-import { AppModule } from '../src/app.module';-import { FeatureManagementService, FeatureType } from '../src/core/features';-import { QuotaService, QuotaType } from '../src/core/quota';+import { AppModule } from '../../src/app.module';+import { FeatureManagementService, FeatureType } from '../../src/core/features';+import { QuotaService, QuotaType } from '../../src/core/quota';import {- checkBlobSize,collectAllBlobSizes,createTestingApp,createWorkspace,@@ -14,7 +13,7 @@ import {listBlobs,setBlob,signUp,-} from './utils';+} from '../utils';const OneMB = 1024 * 1024;@@ -114,58 +113,6 @@ test('should calc all blobs size', async t => {const size = await collectAllBlobSizes(app, u1.token.token);t.is(size, 8, 'failed to collect all blob sizes');-- const size1 = await checkBlobSize(- app,- u1.token.token,- workspace1.id,- 10 * 1024 * 1024 * 1024 - 8- );- t.is(size1, 0, 'failed to check blob size');-- const size2 = await checkBlobSize(- app,- u1.token.token,- workspace1.id,- 10 * 1024 * 1024 * 1024 - 7- );- t.is(size2, -1, 'failed to check blob size');-});--test('should be able calc quota after switch plan', async t => {- const u1 = await signUp(app, 'darksky', '[email protected]', '1');-- const workspace1 = await createWorkspace(app, u1.token.token);-- const buffer1 = Buffer.from([0, 0]);- await setBlob(app, u1.token.token, workspace1.id, buffer1);- const buffer2 = Buffer.from([0, 1]);- await setBlob(app, u1.token.token, workspace1.id, buffer2);-- const workspace2 = await createWorkspace(app, u1.token.token);-- const buffer3 = Buffer.from([0, 0]);- await setBlob(app, u1.token.token, workspace2.id, buffer3);- const buffer4 = Buffer.from([0, 1]);- await setBlob(app, u1.token.token, workspace2.id, buffer4);-- const size1 = await checkBlobSize(- app,- u1.token.token,- workspace1.id,- 10 * 1024 * 1024 * 1024 - 8- );- t.is(size1, 0, 'failed to check free plan blob size');-- await quota.switchUserQuota(u1.id, QuotaType.ProPlanV1);-- const size2 = await checkBlobSize(- app,- u1.token.token,- workspace1.id,- 100 * 1024 * 1024 * 1024 - 8- );- t.is(size2, 0, 'failed to check pro plan blob size');});test('should reject blob exceeded limit', async t => {diff --git a/packages/common/nbstore/package.json b/packages/common/nbstore/package.jsonindex 8d39c3491e..8d0e26c0eb 100644--- a/packages/common/nbstore/package.json+++ b/packages/common/nbstore/package.json@@ -8,7 +8,8 @@".": "./src/index.ts","./op": "./src/op/index.ts","./idb": "./src/impls/idb/index.ts",- "./idb/v1": "./src/impls/idb/v1/index.ts"+ "./idb/v1": "./src/impls/idb/v1/index.ts",+ "./cloud": "./src/impls/cloud/index.ts"},"dependencies": {"@datastructures-js/binary-search-tree": "^5.3.2",@@ -20,11 +21,15 @@"yjs": "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch"},"devDependencies": {+ "@affine/graphql": "workspace:*","fake-indexeddb": "^6.0.0","idb": "^8.0.0",+ "socket.io-client": "^4.7.5","vitest": "2.1.4"},"peerDependencies": {- "idb": "^8.0.0"+ "@affine/graphql": "workspace:*",+ "idb": "^8.0.0",+ "socket.io-client": "^4.7.5"}}diff --git a/packages/common/nbstore/src/impls/cloud/blob.ts b/packages/common/nbstore/src/impls/cloud/blob.tsnew file mode 100644index 0000000000..671687d75c--- /dev/null+++ b/packages/common/nbstore/src/impls/cloud/blob.ts@@ -0,0 +1,72 @@+import {+ deleteBlobMutation,+ gqlFetcherFactory,+ listBlobsQuery,+ releaseDeletedBlobsMutation,+ setBlobMutation,+} from '@affine/graphql';++import { DummyConnection } from '../../connection';+import { type BlobRecord, BlobStorage } from '../../storage';++export class CloudBlobStorage extends BlobStorage {+ private readonly gql = gqlFetcherFactory(this.options.peer + '/graphql');+ override connection = new DummyConnection();++ override async get(key: string) {+ const res = await fetch(+ this.options.peer + '/api/workspaces/' + this.spaceId + '/blobs/' + key,+ { cache: 'default' }+ );++ if (!res.ok) {+ return null;+ }++ const data = await res.arrayBuffer();++ return {+ key,+ data: new Uint8Array(data),+ mime: res.headers.get('content-type') || '',+ size: data.byteLength,+ createdAt: new Date(res.headers.get('last-modified') || Date.now()),+ };+ }++ override async set(blob: BlobRecord) {+ await this.gql({+ query: setBlobMutation,+ variables: {+ workspaceId: this.spaceId,+ blob: new File([blob.data], blob.key, { type: blob.mime }),+ },+ });+ }++ override async delete(key: string, permanently: boolean) {+ await this.gql({+ query: deleteBlobMutation,+ variables: { workspaceId: this.spaceId, key, permanently },+ });+ }++ override async release() {+ await this.gql({+ query: releaseDeletedBlobsMutation,+ variables: { workspaceId: this.spaceId },+ });+ }++ override async list() {+ const res = await this.gql({+ query: listBlobsQuery,+ variables: { workspaceId: this.spaceId },+ });++ return res.workspace.blobs.map(blob => ({+ ...blob,+ createdAt: new Date(blob.createdAt),+ }));+ }+}diff --git a/packages/common/nbstore/src/impls/cloud/doc.ts b/packages/common/nbstore/src/impls/cloud/doc.tsnew file mode 100644index 0000000000..327ac777fa--- /dev/null+++ b/packages/common/nbstore/src/impls/cloud/doc.ts@@ -0,0 +1,199 @@+import { noop } from 'lodash-es';+import type { SocketOptions } from 'socket.io-client';++import { share } from '../../connection';+import {+ type DocClock,+ type DocClocks,+ DocStorage,+ type DocStorageOptions,+ type DocUpdate,+} from '../../storage';+import {+ base64ToUint8Array,+ type ServerEventsMap,+ SocketConnection,+ uint8ArrayToBase64,+} from './socket';++interface CloudDocStorageOptions extends DocStorageOptions {+ socketOptions: SocketOptions;+}++export class CloudDocStorage extends DocStorage<CloudDocStorageOptions> {+ connection = share(+ new SocketConnection(this.peer, this.options.socketOptions)+ );++ private get socket() {+ return this.connection.inner;+ }++ override async connect(): Promise<void> {+ await super.connect();+ this.connection.onStatusChanged(status => {+ if (status === 'connected') {+ this.join().catch(noop);+ this.socket.on('space:broadcast-doc-update', this.onServerUpdate);+ }+ });+ }++ override async disconnect(): Promise<void> {+ this.socket.emit('space:leave', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ });+ this.socket.off('space:broadcast-doc-update', this.onServerUpdate);+ await super.connect();+ }++ async join() {+ try {+ const res = await this.socket.emitWithAck('space:join', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ clientVersion: BUILD_CONFIG.appVersion,+ });++ if ('error' in res) {+ this.connection.setStatus('closed', new Error(res.error.message));+ }+ } catch (e) {+ this.connection.setStatus('error', e as Error);+ }+ }++ onServerUpdate: ServerEventsMap['space:broadcast-doc-update'] = message => {+ if (+ this.spaceType === message.spaceType &&+ this.spaceId === message.spaceId+ ) {+ this.emit('update', {+ docId: message.docId,+ bin: base64ToUint8Array(message.update),+ timestamp: new Date(message.timestamp),+ editor: message.editor,+ });+ }+ };++ override async getDocSnapshot(docId: string) {+ const response = await this.socket.emitWithAck('space:load-doc', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ docId,+ });++ if ('error' in response) {+ // TODO: use [UserFriendlyError]+ throw new Error(response.error.message);+ }++ return {+ docId,+ bin: base64ToUint8Array(response.data.missing),+ timestamp: new Date(response.data.timestamp),+ };+ }++ override async getDocDiff(docId: string, state?: Uint8Array) {+ const response = await this.socket.emitWithAck('space:load-doc', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ docId,+ stateVector: state ? await uint8ArrayToBase64(state) : void 0,+ });++ if ('error' in response) {+ // TODO: use [UserFriendlyError]+ throw new Error(response.error.message);+ }++ return {+ docId,+ missing: base64ToUint8Array(response.data.missing),+ state: base64ToUint8Array(response.data.state),+ timestamp: new Date(response.data.timestamp),+ };+ }++ override async pushDocUpdate(update: DocUpdate) {+ const response = await this.socket.emitWithAck('space:push-doc-update', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ docId: update.docId,+ updates: await uint8ArrayToBase64(update.bin),+ });++ if ('error' in response) {+ // TODO(@forehalo): use [UserFriendlyError]+ throw new Error(response.error.message);+ }++ return {+ docId: update.docId,+ timestamp: new Date(response.data.timestamp),+ };+ }++ /**+ * Just a rough implementation, cloud doc storage should not need this method.+ */+ override async getDocTimestamp(docId: string): Promise<DocClock | null> {+ const response = await this.socket.emitWithAck('space:load-doc', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ docId,+ });++ if ('error' in response) {+ // TODO: use [UserFriendlyError]+ throw new Error(response.error.message);+ }++ return {+ docId,+ timestamp: new Date(response.data.timestamp),+ };+ }++ override async getDocTimestamps(after?: Date) {+ const response = await this.socket.emitWithAck(+ 'space:load-doc-timestamps',+ {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ timestamp: after ? after.getTime() : undefined,+ }+ );++ if ('error' in response) {+ // TODO(@forehalo): use [UserFriendlyError]+ throw new Error(response.error.message);+ }++ return Object.entries(response.data).reduce((ret, [docId, timestamp]) => {+ ret[docId] = new Date(timestamp);+ return ret;+ }, {} as DocClocks);+ }++ override async deleteDoc(docId: string) {+ this.socket.emit('space:delete-doc', {+ spaceType: this.spaceType,+ spaceId: this.spaceId,+ docId,+ });+ }++ protected async setDocSnapshot() {+ return false;+ }+ protected async getDocUpdates() {+ return [];+ }+ protected async markUpdatesMerged() {+ return 0;+ }+}diff --git a/packages/common/nbstore/src/impls/cloud/index.ts b/packages/common/nbstore/src/impls/cloud/index.tsnew file mode 100644index 0000000000..d476ae6eb9--- /dev/null+++ b/packages/common/nbstore/src/impls/cloud/index.ts@@ -0,0 +1,2 @@+export * from './blob';+export * from './doc';diff --git a/packages/common/nbstore/src/impls/cloud/socket.ts b/packages/common/nbstore/src/impls/cloud/socket.tsnew file mode 100644index 0000000000..6e84a433e3--- /dev/null+++ b/packages/common/nbstore/src/impls/cloud/socket.ts@@ -0,0 +1,173 @@+import {+ Manager as SocketIOManager,+ type Socket as SocketIO,+ type SocketOptions,+} from 'socket.io-client';++import { Connection, type ConnectionStatus } from '../../connection';++// TODO(@forehalo): use [UserFriendlyError]+interface EventError {+ name: string;+ message: string;+}++type WebsocketResponse<T> =+ | {+ error: EventError;+ }+ | {+ data: T;+ };++interface ServerEvents {+ 'space:broadcast-doc-update': {+ spaceType: string;+ spaceId: string;+ docId: string;+ update: string;+ timestamp: number;+ editor: string;+ };+}++interface ClientEvents {+ 'space:join': [+ { spaceType: string; spaceId: string; clientVersion: string },+ { clientId: string },+ ];+ 'space:leave': { spaceType: string; spaceId: string };+ 'space:join-awareness': [+ {+ spaceType: string;+ spaceId: string;+ docId: string;+ clientVersion: string;+ },+ { clientId: string },+ ];+ 'space:leave-awareness': {+ spaceType: string;+ spaceId: string;+ docId: string;+ };++ 'space:push-doc-update': [+ { spaceType: string; spaceId: string; docId: string; updates: string },+ { timestamp: number },+ ];+ 'space:load-doc-timestamps': [+ {+ spaceType: string;+ spaceId: string;+ timestamp?: number;+ },+ Record<string, number>,+ ];+ 'space:load-doc': [+ {+ spaceType: string;+ spaceId: string;+ docId: string;+ stateVector?: string;+ },+ {+ missing: string;+ state: string;+ timestamp: number;+ },+ ];+ 'space:delete-doc': { spaceType: string; spaceId: string; docId: string };+}++export type ServerEventsMap = {+ [Key in keyof ServerEvents]: (data: ServerEvents[Key]) => void;+};++export type ClientEventsMap = {+ [Key in keyof ClientEvents]: ClientEvents[Key] extends Array<any>+ ? (+ data: ClientEvents[Key][0],+ ack: (res: WebsocketResponse<ClientEvents[Key][1]>) => void+ ) => void+ : (data: ClientEvents[Key]) => void;+};++export type Socket = SocketIO<ServerEventsMap, ClientEventsMap>;++export function uint8ArrayToBase64(array: Uint8Array): Promise<string> {+ return new Promise<string>(resolve => {+ // Create a blob from the Uint8Array+ const blob = new Blob([array]);++ const reader = new FileReader();+ reader.onload = function () {+ const dataUrl = reader.result as string | null;+ if (!dataUrl) {+ resolve('');+ return;+ }+ // The result includes the `data:` URL prefix and the MIME type. We only want the Base64 data+ const base64 = dataUrl.split(',')[1];+ resolve(base64);+ };++ reader.readAsDataURL(blob);+ });+}++export function base64ToUint8Array(base64: string) {+ const binaryString = atob(base64);+ const binaryArray = binaryString.split('').map(function (char) {+ return char.charCodeAt(0);+ });+ return new Uint8Array(binaryArray);+}++export class SocketConnection extends Connection<Socket> {+ manager = new SocketIOManager(this.endpoint, {+ autoConnect: false,+ transports: ['websocket'],+ secure: new URL(this.endpoint).protocol === 'https:',+ });++ constructor(+ private readonly endpoint: string,+ private readonly socketOptions: SocketOptions+ ) {+ super();+ }++ override get shareId() {+ return `socket:${this.endpoint}`;+ }++ override async doConnect() {+ const conn = this.manager.socket('/', this.socketOptions);++ await new Promise<void>((resolve, reject) => {+ conn.once('connect', () => {+ resolve();+ });+ conn.once('connect_error', err => {+ reject(err);+ });+ conn.open();+ });++ return conn;+ }++ override async doDisconnect(conn: Socket) {+ conn.close();+ }++ /**+ * Socket connection allow explicitly set status by user+ *+ * used when join space failed+ */+ override setStatus(status: ConnectionStatus, error?: Error) {+ super.setStatus(status, error);+ }+}diff --git a/packages/common/nbstore/src/impls/index.ts b/packages/common/nbstore/src/impls/index.tsindex 8bce204798..bff2874ad4 100644--- a/packages/common/nbstore/src/impls/index.ts+++ b/packages/common/nbstore/src/impls/index.ts@@ -1,4 +1,5 @@import type { Storage } from '../storage';+import { CloudBlobStorage, CloudDocStorage } from './cloud';import {IndexedDBBlobStorage,IndexedDBDocStorage,@@ -19,7 +20,9 @@ const idbv1: StorageConstructor[] = [IndexedDBV1BlobStorage,];-export const storages: StorageConstructor[] = [...idbv1, ...idb];+const cloud: StorageConstructor[] = [CloudDocStorage, CloudBlobStorage];++export const storages: StorageConstructor[] = cloud.concat(idbv1, idb);const AvailableStorageImplementations = storages.reduce((acc, curr) => {diff --git a/packages/frontend/core/src/modules/cloud/stores/user-quota.ts b/packages/frontend/core/src/modules/cloud/stores/user-quota.tsindex 0413d46dcb..70698060ae 100644--- a/packages/frontend/core/src/modules/cloud/stores/user-quota.ts+++ b/packages/frontend/core/src/modules/cloud/stores/user-quota.ts@@ -23,7 +23,7 @@ export class UserQuotaStore extends Store {return {userId: data.currentUser.id,quota: data.currentUser.quota,- used: data.collectAllBlobSizes.size,+ used: data.currentUser.quotaUsage.storageQuota,};}}diff --git a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.tsindex 3eacf5dbfb..e6549b0e05 100644--- a/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts+++ b/packages/frontend/core/src/modules/workspace-engine/impls/engine/blob-cloud.ts@@ -70,7 +70,7 @@ export class CloudBlobStorage implements BlobStorage {query: deleteBlobMutation,variables: {workspaceId: key,- hash: key,+ key,},});}@@ -82,6 +82,6 @@ export class CloudBlobStorage implements BlobStorage {workspaceId: this.workspaceId,},});- return result.listBlobs;+ return result.workspace.blobs.map(blob => blob.key);}}diff --git a/packages/frontend/graphql/src/graphql/blob-delete.gql b/packages/frontend/graphql/src/graphql/blob-delete.gqlindex 790c326465..ab78a54d88 100644--- a/packages/frontend/graphql/src/graphql/blob-delete.gql+++ b/packages/frontend/graphql/src/graphql/blob-delete.gql@@ -1,3 +1,7 @@-mutation deleteBlob($workspaceId: String!, $hash: String!) {- deleteBlob(workspaceId: $workspaceId, hash: $hash)+mutation deleteBlob(+ $workspaceId: String!+ $key: String!+ $permanently: Boolean+) {+ deleteBlob(workspaceId: $workspaceId, key: $key, permanently: $permanently)}diff --git a/packages/frontend/graphql/src/graphql/blob-list.gql b/packages/frontend/graphql/src/graphql/blob-list.gqlindex 67a19c718c..5b314f0cc5 100644--- a/packages/frontend/graphql/src/graphql/blob-list.gql+++ b/packages/frontend/graphql/src/graphql/blob-list.gql@@ -1,3 +1,10 @@query listBlobs($workspaceId: String!) {- listBlobs(workspaceId: $workspaceId)+ workspace(id: $workspaceId) {+ blobs {+ key+ size+ mime+ createdAt+ }+ }}diff --git a/packages/frontend/graphql/src/graphql/blob-release-deleted.gql b/packages/frontend/graphql/src/graphql/blob-release-deleted.gqlnew file mode 100644index 0000000000..d51d0ef7ca--- /dev/null+++ b/packages/frontend/graphql/src/graphql/blob-release-deleted.gql@@ -0,0 +1,3 @@+mutation releaseDeletedBlobs($workspaceId: String!) {+ releaseDeletedBlobs(workspaceId: $workspaceId)+}diff --git a/packages/frontend/graphql/src/graphql/index.ts b/packages/frontend/graphql/src/graphql/index.tsindex 638cfd3846..d22e5dc345 100644--- a/packages/frontend/graphql/src/graphql/index.ts+++ b/packages/frontend/graphql/src/graphql/index.ts@@ -47,19 +47,37 @@ export const deleteBlobMutation = {definitionName: 'deleteBlob',containsFile: false,query: `-mutation deleteBlob($workspaceId: String!, $hash: String!) {- deleteBlob(workspaceId: $workspaceId, hash: $hash)+mutation deleteBlob($workspaceId: String!, $key: String!, $permanently: Boolean) {+ deleteBlob(workspaceId: $workspaceId, key: $key, permanently: $permanently)}`,};export const listBlobsQuery = {id: 'listBlobsQuery' as const,operationName: 'listBlobs',- definitionName: 'listBlobs',+ definitionName: 'workspace',containsFile: false,query: `query listBlobs($workspaceId: String!) {- listBlobs(workspaceId: $workspaceId)+ workspace(id: $workspaceId) {+ blobs {+ key+ size+ mime+ createdAt+ }+ }+}`,+};++export const releaseDeletedBlobsMutation = {+ id: 'releaseDeletedBlobsMutation' as const,+ operationName: 'releaseDeletedBlobs',+ definitionName: 'releaseDeletedBlobs',+ containsFile: false,+ query: `+mutation releaseDeletedBlobs($workspaceId: String!) {+ releaseDeletedBlobs(workspaceId: $workspaceId)}`,};@@ -885,7 +903,7 @@ mutation publishPage($workspaceId: String!, $pageId: String!, $mode: PublicPageMexport const quotaQuery = {id: 'quotaQuery' as const,operationName: 'quota',- definitionName: 'currentUser,collectAllBlobSizes',+ definitionName: 'currentUser',containsFile: false,query: `query quota {@@ -905,9 +923,9 @@ query quota {memberLimit}}- }- collectAllBlobSizes {- size+ quotaUsage {+ storageQuota+ }}}`,};diff --git a/packages/frontend/graphql/src/graphql/quota.gql b/packages/frontend/graphql/src/graphql/quota.gqlindex c0268b6443..682c2e5bc3 100644--- a/packages/frontend/graphql/src/graphql/quota.gql+++ b/packages/frontend/graphql/src/graphql/quota.gql@@ -15,8 +15,8 @@ query quota {memberLimit}}- }- collectAllBlobSizes {- size+ quotaUsage {+ storageQuota+ }}}diff --git a/packages/frontend/graphql/src/schema.ts b/packages/frontend/graphql/src/schema.tsindex 4a7fd44c55..4c554e068e 100644--- a/packages/frontend/graphql/src/schema.ts+++ b/packages/frontend/graphql/src/schema.ts@@ -517,6 +517,14 @@ export interface ListUserInput {skip?: InputMaybe<Scalars['Int']['input']>;}+export interface ListedBlob {+ __typename?: 'ListedBlob';+ createdAt: Scalars['String']['output'];+ key: Scalars['String']['output'];+ mime: Scalars['String']['output'];+ size: Scalars['Int']['output'];+}+export interface ManageUserInput {/** User email */email?: InputMaybe<Scalars['String']['input']>;@@ -569,6 +577,7 @@ export interface Mutation {leaveWorkspace: Scalars['Boolean']['output'];publishPage: WorkspacePage;recoverDoc: Scalars['DateTime']['output'];+ releaseDeletedBlobs: Scalars['Boolean']['output'];/** Remove user avatar */removeAvatar: RemoveAvatar;removeWorkspaceFeature: Scalars['Int']['output'];@@ -673,7 +682,9 @@ export interface MutationCreateWorkspaceArgs {}export interface MutationDeleteBlobArgs {- hash: Scalars['String']['input'];+ hash?: InputMaybe<Scalars['String']['input']>;+ key?: InputMaybe<Scalars['String']['input']>;+ permanently?: Scalars['Boolean']['input'];workspaceId: Scalars['String']['input'];}@@ -731,6 +742,10 @@ export interface MutationRecoverDocArgs {workspaceId: Scalars['String']['input'];}+export interface MutationReleaseDeletedBlobsArgs {+ workspaceId: Scalars['String']['input'];+}+export interface MutationRemoveWorkspaceFeatureArgs {feature: FeatureType;workspaceId: Scalars['String']['input'];@@ -882,9 +897,7 @@ export enum PublicPageMode {export interface Query {__typename?: 'Query';- /** @deprecated no more needed */- checkBlobSize: WorkspaceBlobSizes;- /** @deprecated use `user.storageUsage` instead */+ /** @deprecated use `user.quotaUsage` instead */collectAllBlobSizes: WorkspaceBlobSizes;/** Get current user */currentUser: Maybe<UserType>;@@ -925,11 +938,6 @@ export interface Query {workspaces: Array<WorkspaceType>;}-export interface QueryCheckBlobSizeArgs {- size: Scalars['SafeInt']['input'];- workspaceId: Scalars['String']['input'];-}-export interface QueryErrorArgs {name: ErrorNames;}@@ -1225,6 +1233,11 @@ export interface UserQuotaHumanReadable {storageQuota: Scalars['String']['output'];}+export interface UserQuotaUsage {+ __typename?: 'UserQuotaUsage';+ storageQuota: Scalars['SafeInt']['output'];+}+export interface UserType {__typename?: 'UserType';/** User avatar url */@@ -1250,6 +1263,7 @@ export interface UserType {/** User name */name: Scalars['String']['output'];quota: Maybe<UserQuota>;+ quotaUsage: UserQuotaUsage;subscriptions: Array<SubscriptionType>;/** @deprecated use [/api/auth/sign-in?native=true] instead */token: TokenType;@@ -1313,7 +1327,7 @@ export interface WorkspaceType {/** Available features of workspace */availableFeatures: Array<FeatureType>;/** List blobs of workspace */- blobs: Array<Scalars['String']['output']>;+ blobs: Array<ListedBlob>;/** Blobs size of workspace */blobsSize: Scalars['Int']['output'];/** Workspace created date */@@ -1417,7 +1431,8 @@ export type AdminServerConfigQuery = {export type DeleteBlobMutationVariables = Exact<{workspaceId: Scalars['String']['input'];- hash: Scalars['String']['input'];+ key: Scalars['String']['input'];+ permanently?: InputMaybe<Scalars['Boolean']['input']>;}>;export type DeleteBlobMutation = {@@ -1429,7 +1444,28 @@ export type ListBlobsQueryVariables = Exact<{workspaceId: Scalars['String']['input'];}>;-export type ListBlobsQuery = { __typename?: 'Query'; listBlobs: Array<string> };+export type ListBlobsQuery = {+ __typename?: 'Query';+ workspace: {+ __typename?: 'WorkspaceType';+ blobs: Array<{+ __typename?: 'ListedBlob';+ key: string;+ size: number;+ mime: string;+ createdAt: string;+ }>;+ };+};++export type ReleaseDeletedBlobsMutationVariables = Exact<{+ workspaceId: Scalars['String']['input'];+}>;++export type ReleaseDeletedBlobsMutation = {+ __typename?: 'Mutation';+ releaseDeletedBlobs: boolean;+};export type SetBlobMutationVariables = Exact<{workspaceId: Scalars['String']['input'];@@ -2196,8 +2232,8 @@ export type QuotaQuery = {memberLimit: string;};} | null;+ quotaUsage: { __typename?: 'UserQuotaUsage'; storageQuota: number };} | null;- collectAllBlobSizes: { __typename?: 'WorkspaceBlobSizes'; size: number };};export type RecoverDocMutationVariables = Exact<{@@ -2953,6 +2989,11 @@ export type Mutations =variables: DeleteBlobMutationVariables;response: DeleteBlobMutation;}+ | {+ name: 'releaseDeletedBlobsMutation';+ variables: ReleaseDeletedBlobsMutationVariables;+ response: ReleaseDeletedBlobsMutation;+ }| {name: 'setBlobMutation';variables: SetBlobMutationVariables;diff --git a/yarn.lock b/yarn.lockindex 784182955d..0a341daab0 100644--- a/yarn.lock+++ b/yarn.lock@@ -722,6 +722,7 @@ __metadata:version: 0.0.0-use.localresolution: "@affine/nbstore@workspace:packages/common/nbstore"dependencies:+ "@affine/graphql": "workspace:*""@datastructures-js/binary-search-tree": "npm:^5.3.2""@toeverything/infra": "workspace:*"eventemitter2: "npm:^6.4.9"@@ -730,10 +731,13 @@ __metadata:lodash-es: "npm:^4.17.21"nanoid: "npm:^5.0.7"rxjs: "npm:^7.8.1"+ socket.io-client: "npm:^4.7.5"vitest: "npm:2.1.4"yjs: "patch:yjs@npm%3A13.6.18#~/.yarn/patches/yjs-npm-13.6.18-ad0d5f7c43.patch"peerDependencies:+ "@affine/graphql": "workspace:*"idb: ^8.0.0+ socket.io-client: ^4.7.5languageName: unknownlinkType: soft@@ -11491,32 +11495,42 @@ __metadata:languageName: nodelinkType: hard-"@shikijs/core@npm:1.24.0":- version: 1.24.0- resolution: "@shikijs/core@npm:1.24.0"+"@shikijs/core@npm:1.22.2":+ version: 1.22.2+ resolution: "@shikijs/core@npm:1.22.2"dependencies:- "@shikijs/engine-javascript": "npm:1.24.0"- "@shikijs/engine-oniguruma": "npm:1.24.0"- "@shikijs/types": "npm:1.24.0"+ "@shikijs/engine-javascript": "npm:1.22.2"+ "@shikijs/engine-oniguruma": "npm:1.22.2"+ "@shikijs/types": "npm:1.22.2""@shikijs/vscode-textmate": "npm:^9.3.0""@types/hast": "npm:^3.0.4"hast-util-to-html: "npm:^9.0.3"- checksum: 10/7a8944676d37c902b83b0585164675d55185881079bb75ce5411cb01fe6f4c5fd8411291e99a399c2d9f6b21955fbb6a7e3040e16f8bf43b2bafaa033320e12f+ checksum: 10/c5203e1cbef8e159fc4ef4556b350dc055d9d15af57cb12ea699c94ecd603e58f7000e106fd5d103e1a1c8d1cc975cd7c573e9bacaa01f7e5eaa05b921d1ee38languageName: nodelinkType: hard-"@shikijs/engine-javascript@npm:1.24.0":- version: 1.24.0- resolution: "@shikijs/engine-javascript@npm:1.24.0"+"@shikijs/engine-javascript@npm:1.22.2":+ version: 1.22.2+ resolution: "@shikijs/engine-javascript@npm:1.22.2"dependencies:- "@shikijs/types": "npm:1.24.0"+ "@shikijs/types": "npm:1.22.2""@shikijs/vscode-textmate": "npm:^9.3.0"- oniguruma-to-es: "npm:0.7.0"- checksum: 10/f5a1832bcbad0761292172dbfdd4dbedbd87f9f31f786606798228d5355ccb8d10bf672741a6e1d2be48570b8cf810402704a4131e82ac10797806b5301e34d8+ oniguruma-to-js: "npm:0.4.3"+ checksum: 10/162f089f7ec7bc8e6877e1047bdf339a7446b7407ad0bffcb4b7372263ae5aae0be429f1c87054326be79d4e1bbe55849c010ea4aa499e83816ce009e490938blanguageName: nodelinkType: hard-"@shikijs/engine-oniguruma@npm:1.24.0, @shikijs/engine-oniguruma@npm:^1.24.0":+"@shikijs/engine-oniguruma@npm:1.22.2":+ version: 1.22.2+ resolution: "@shikijs/engine-oniguruma@npm:1.22.2"+ dependencies:+ "@shikijs/types": "npm:1.22.2"+ "@shikijs/vscode-textmate": "npm:^9.3.0"+ checksum: 10/924fff6c3d0e464ab2bde326076535fc1f98c0c90ceba1811b89f25c7b9df36a0fbec509b2859f5e2097e29f8087e90fa6b951fa8183d44c1abaa21a49c71e9e+ languageName: node+ linkType: hard++"@shikijs/engine-oniguruma@npm:^1.24.0":version: 1.24.0resolution: "@shikijs/engine-oniguruma@npm:1.24.0"dependencies:@@ -11526,6 +11540,16 @@ __metadata:languageName: nodelinkType: hard+"@shikijs/types@npm:1.22.2":+ version: 1.22.2+ resolution: "@shikijs/types@npm:1.22.2"+ dependencies:+ "@shikijs/vscode-textmate": "npm:^9.3.0"+ "@types/hast": "npm:^3.0.4"+ checksum: 10/bba6e4d8ef76fae30e9c298539e6b6b4f82360d894352fc54882531e71b5f5c490a1a49ae83d4133a0de85c7d58ec678c6ceb5f5f5d012cd09a289d8845b1737+ languageName: node+ linkType: hard+"@shikijs/types@npm:1.24.0, @shikijs/types@npm:^1.24.0":version: 1.24.0resolution: "@shikijs/types@npm:1.24.0"@@ -18915,13 +18939,6 @@ __metadata:languageName: nodelinkType: hard-"emoji-regex-xs@npm:^1.0.0":- version: 1.0.0- resolution: "emoji-regex-xs@npm:1.0.0"- checksum: 10/e216ec4270f765e1097cefc1b9518a7166b872b4424c60a85d79765f318d989cd458e036c76c13e9ce2ed1fe1bb5935a7fd5c1fab7600668bc8e92a789045b3c- languageName: node- linkType: hard-"emoji-regex@npm:^10.3.0":version: 10.4.0resolution: "emoji-regex@npm:10.4.0"@@ -26468,14 +26485,12 @@ __metadata:languageName: nodelinkType: hard-"oniguruma-to-es@npm:0.7.0":- version: 0.7.0- resolution: "oniguruma-to-es@npm:0.7.0"+"oniguruma-to-js@npm:0.4.3":+ version: 0.4.3+ resolution: "oniguruma-to-js@npm:0.4.3"dependencies:- emoji-regex-xs: "npm:^1.0.0"- regex: "npm:^5.0.2"- regex-recursion: "npm:^4.3.0"- checksum: 10/766f2c4a9a9eb97070914ebbd78517d073c58f2558994cffb58b064facf860b8f568c7146281e527c796631e93ac23cc1c4b897436189033785429a4486ad41d+ regex: "npm:^4.3.2"+ checksum: 10/af64a77f4e428c2652387014596138c51bd61d67b0bbe957cd10ff73b4ec14567701ff9286342ab804cfa00486a9a0ff189da8391721c21c898ea8e26b62e74flanguageName: nodelinkType: hard@@ -28783,28 +28798,10 @@ __metadata:languageName: nodelinkType: hard-"regex-recursion@npm:^4.3.0":- version: 4.3.0- resolution: "regex-recursion@npm:4.3.0"- dependencies:- regex-utilities: "npm:^2.3.0"- checksum: 10/bbb7fcd6542c980cb3a4571186928826b263759e89bbc1c7b313d9f1064b6b1878c414a696b9cee01156a42225e508a62003f3edaab52a0a3344debf3211ebd8- languageName: node- linkType: hard--"regex-utilities@npm:^2.3.0":- version: 2.3.0- resolution: "regex-utilities@npm:2.3.0"- checksum: 10/d11519c31f379488cbc6278b8645d72f16339ee325c79a4b8b3a6477738016a52983158dc69ae1b5867f8b06978ff5d83933520257a57f7e5c3e4ac6a1ea3cc7- languageName: node- linkType: hard--"regex@npm:^5.0.2":- version: 5.0.2- resolution: "regex@npm:5.0.2"- dependencies:- regex-utilities: "npm:^2.3.0"- checksum: 10/c9dab5adc2df30a37bed0665b4830be170e413e48bb0fc149388161995dc250049ce0aa5e579757b3c6c0ecb8cb2b9afe50d3a5de229cbed36132ff9cc93efa6+"regex@npm:^4.3.2":+ version: 4.4.0+ resolution: "regex@npm:4.4.0"+ checksum: 10/0a32dcf2913287f5225a48aff11d26423711734307135c4dde489e71be35939b0d1fc253ddb9d81549e27451a4346a6401e87c10b1f4567fb928ad67279fbe31languageName: nodelinkType: hard@@ -29976,16 +29973,16 @@ __metadata:linkType: hard"shiki@npm:^1.12.0, shiki@npm:^1.14.1":- version: 1.24.0- resolution: "shiki@npm:1.24.0"+ version: 1.22.2+ resolution: "shiki@npm:1.22.2"dependencies:- "@shikijs/core": "npm:1.24.0"- "@shikijs/engine-javascript": "npm:1.24.0"- "@shikijs/engine-oniguruma": "npm:1.24.0"- "@shikijs/types": "npm:1.24.0"+ "@shikijs/core": "npm:1.22.2"+ "@shikijs/engine-javascript": "npm:1.22.2"+ "@shikijs/engine-oniguruma": "npm:1.22.2"+ "@shikijs/types": "npm:1.22.2""@shikijs/vscode-textmate": "npm:^9.3.0""@types/hast": "npm:^3.0.4"- checksum: 10/1e4f119fcd365a3ebfe50dec3d9c5f40672f4530b1b42587e03edbffb77395dcf8e4e70dccf1fd9b68eeff5769aa7d1cfdb7ea139e8312c1f58507c5d222a2c5+ checksum: 10/5da1925609662cc773a807c3e9223805dc3323eaf2081aaf6633f3c20846485cab1429601f0a8a4a4aab83c80382b5b03ea9c94edffd3db968ab68de2484315clanguageName: nodelinkType: hard