There can be more than Notion and Miro. AFFiNE(pronounced [ə‘fain]) is a next-gen knowledge base that brings planning, sorting and creating all together. Privacy first, open-source, customizable and ready to use.
TypeScript appcrdteditorelectronknowledge-basemarkdownmironotesnotionnotion-alternativerustrust-langrust-languagetabletableviewwhiteboardwikiworkspace
feat: add blob upload support for copilot (#6584)
DarkSkyccb3bed91ee8
diff --git a/packages/backend/server/src/core/quota/storage.ts b/packages/backend/server/src/core/quota/storage.tsindex 7ec0d510e6..8ba20532bb 100644--- a/packages/backend/server/src/core/quota/storage.ts+++ b/packages/backend/server/src/core/quota/storage.ts@@ -7,7 +7,10 @@ import { OneGB } from './constant';import { QuotaService } from './service';import { formatSize, QuotaQueryType } from './types';-type QuotaBusinessType = QuotaQueryType & { businessBlobLimit: number };+type QuotaBusinessType = QuotaQueryType & {+ businessBlobLimit: number;+ unlimited: boolean;+};@Injectable()export class QuotaManagementService {@@ -59,6 +62,52 @@ export class QuotaManagementService {}, 0);}+ private generateQuotaCalculator(+ quota: number,+ blobLimit: number,+ usedSize: number,+ unlimited = false+ ) {+ const checkExceeded = (recvSize: number) => {+ const total = usedSize + recvSize;+ // only skip total storage check if workspace has unlimited feature+ if (total > quota && !unlimited) {+ this.logger.log(`storage size limit exceeded: ${total} > ${quota}`);+ return true;+ } else if (recvSize > blobLimit) {+ this.logger.log(`blob size limit exceeded: ${recvSize} > ${blobLimit}`);+ return true;+ } else {+ return false;+ }+ };+ return checkExceeded;+ }++ async getQuotaCalculator(userId: string) {+ const quota = await this.getUserQuota(userId);+ const { storageQuota, businessBlobLimit } = quota;+ const usedSize = await this.getUserUsage(userId);++ return this.generateQuotaCalculator(+ storageQuota,+ businessBlobLimit,+ usedSize+ );+ }++ async getQuotaCalculatorByWorkspace(workspaceId: string) {+ const { storageQuota, usedSize, businessBlobLimit, unlimited } =+ await this.getWorkspaceUsage(workspaceId);++ return this.generateQuotaCalculator(+ storageQuota,+ businessBlobLimit,+ usedSize,+ unlimited+ );+ }+// get workspace's owner quota and total size of used// quota was apply to owner's accountasync getWorkspaceUsage(workspaceId: string): Promise<QuotaBusinessType> {@@ -79,6 +128,12 @@ export class QuotaManagementService {} = await this.quota.getUserQuota(owner.id);// get all workspaces size of owner usedconst usedSize = await this.getUserUsage(owner.id);+ // relax restrictions if workspace has unlimited feature+ // todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota+ const unlimited = await this.feature.hasWorkspaceFeature(+ workspaceId,+ FeatureType.UnlimitedWorkspace+ );const quota = {name,@@ -90,15 +145,10 @@ export class QuotaManagementService {copilotActionLimit,humanReadable,usedSize,+ unlimited,};- // relax restrictions if workspace has unlimited feature- // todo(@darkskygit): need a mechanism to allow feature as a middleware to edit quota- const unlimited = await this.feature.hasWorkspaceFeature(- workspaceId,- FeatureType.UnlimitedWorkspace- );- if (unlimited) {+ if (quota.unlimited) {return this.mergeUnlimitedQuota(quota);}diff --git a/packages/backend/server/src/core/workspaces/resolvers/blob.ts b/packages/backend/server/src/core/workspaces/resolvers/blob.tsindex 335fea36bb..9717dddb7c 100644--- a/packages/backend/server/src/core/workspaces/resolvers/blob.ts+++ b/packages/backend/server/src/core/workspaces/resolvers/blob.ts@@ -1,8 +1,4 @@-import {- ForbiddenException,- Logger,- PayloadTooLargeException,-} from '@nestjs/common';+import { Logger, PayloadTooLargeException, UseGuards } from '@nestjs/common';import {Args,Int,@@ -16,20 +12,23 @@ import { SafeIntResolver } from 'graphql-scalars';import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';import type { FileUpload } from '../../../fundamentals';-import { MakeCache, PreventCache } from '../../../fundamentals';+import {+ CloudThrottlerGuard,+ MakeCache,+ PreventCache,+} from '../../../fundamentals';import { CurrentUser } from '../../auth';-import { FeatureManagementService, FeatureType } from '../../features';import { QuotaManagementService } from '../../quota';import { WorkspaceBlobStorage } from '../../storage';import { PermissionService } from '../permission';import { Permission, WorkspaceBlobSizes, WorkspaceType } from '../types';+@UseGuards(CloudThrottlerGuard)@Resolver(() => WorkspaceType)export class WorkspaceBlobResolver {logger = new Logger(WorkspaceBlobResolver.name);constructor(private readonly permissions: PermissionService,- private readonly feature: FeatureManagementService,private readonly quota: QuotaManagementService,private readonly storage: WorkspaceBlobStorage) {}@@ -124,34 +123,8 @@ export class WorkspaceBlobResolver {Permission.Write);- const { storageQuota, usedSize, businessBlobLimit } =- await this.quota.getWorkspaceUsage(workspaceId);-- const unlimited = await this.feature.hasWorkspaceFeature(- workspaceId,- FeatureType.UnlimitedWorkspace- );-- const checkExceeded = (recvSize: number) => {- if (!storageQuota) {- throw new ForbiddenException('Cannot find user quota.');- }- const total = usedSize + recvSize;- // only skip total storage check if workspace has unlimited feature- if (total > storageQuota && !unlimited) {- this.logger.log(- `storage size limit exceeded: ${total} > ${storageQuota}`- );- return true;- } else if (recvSize > businessBlobLimit) {- this.logger.log(- `blob size limit exceeded: ${recvSize} > ${businessBlobLimit}`- );- return true;- } else {- return false;- }- };+ const checkExceeded =+ await this.quota.getQuotaCalculatorByWorkspace(workspaceId);if (checkExceeded(0)) {throw new PayloadTooLargeException(diff --git a/packages/backend/server/src/fundamentals/config/storage/index.ts b/packages/backend/server/src/fundamentals/config/storage/index.tsindex 32f989f86c..a56404277f 100644--- a/packages/backend/server/src/fundamentals/config/storage/index.ts+++ b/packages/backend/server/src/fundamentals/config/storage/index.ts@@ -19,6 +19,7 @@ export type StorageConfig<Ext = unknown> = {export interface StoragesConfig {avatar: StorageConfig<{ publicLinkFactory: (key: string) => string }>;blob: StorageConfig;+ copilot: StorageConfig;}export interface AFFiNEStorageConfig {@@ -51,6 +52,10 @@ export function getDefaultAFFiNEStorageConfig(): AFFiNEStorageConfig {provider: 'fs',bucket: 'blobs',},+ copilot: {+ provider: 'fs',+ bucket: 'copilot',+ },},};}diff --git a/packages/backend/server/src/plugins/copilot/controller.ts b/packages/backend/server/src/plugins/copilot/controller.tsindex bb5d128b6d..d78c1a2105 100644--- a/packages/backend/server/src/plugins/copilot/controller.ts+++ b/packages/backend/server/src/plugins/copilot/controller.ts@@ -3,6 +3,8 @@ import {Controller,Get,InternalServerErrorException,+ Logger,+ NotFoundException,Param,Query,Req,@@ -17,15 +19,18 @@ import {from,map,merge,+ mergeMap,Observable,switchMap,toArray,} from 'rxjs';+import { Public } from '../../core/auth';import { CurrentUser } from '../../core/auth/current-user';import { Config } from '../../fundamentals';import { CopilotProviderService } from './providers';import { ChatSession, ChatSessionService } from './session';+import { CopilotStorage } from './storage';import { CopilotCapability } from './types';export interface ChatEvent {@@ -36,10 +41,13 @@ export interface ChatEvent {@Controller('/api/copilot')export class CopilotController {+ private readonly logger = new Logger(CopilotController.name);+constructor(private readonly config: Config,private readonly chatSession: ChatSessionService,- private readonly provider: CopilotProviderService+ private readonly provider: CopilotProviderService,+ private readonly storage: CopilotStorage) {}private async hasAttachment(sessionId: string, messageId?: string) {@@ -230,12 +238,19 @@ export class CopilotController {delete params.message;delete params.messageId;+ const handleRemoteLink = this.storage.handleRemoteLink.bind(+ this.storage,+ user.id,+ sessionId+ );+return from(provider.generateImagesStream(session.finish(params), session.model, {signal: this.getSignal(req),user: user.id,})).pipe(+ mergeMap(handleRemoteLink),connect(shared$ =>merge(// actual chat event stream@@ -294,4 +309,33 @@ export class CopilotController {res.status(response.status).send(await response.json());}++ @Public()+ @Get('/blob/:userId/:workspaceId/:key')+ async getBlob(+ @Res() res: Response,+ @Param('userId') userId: string,+ @Param('workspaceId') workspaceId: string,+ @Param('key') key: string+ ) {+ const { body, metadata } = await this.storage.get(userId, workspaceId, key);++ if (!body) {+ throw new NotFoundException(+ `Blob not found in ${userId}'s workspace ${workspaceId}: ${key}`+ );+ }++ // metadata should always exists if body is not null+ if (metadata) {+ res.setHeader('content-type', metadata.contentType);+ res.setHeader('last-modified', metadata.lastModified.toUTCString());+ res.setHeader('content-length', metadata.contentLength);+ } else {+ this.logger.warn(`Blob ${workspaceId}/${key} has no metadata`);+ }++ res.setHeader('cache-control', 'public, max-age=2592000, immutable');+ body.pipe(res);+ }}diff --git a/packages/backend/server/src/plugins/copilot/index.ts b/packages/backend/server/src/plugins/copilot/index.tsindex 6d65f5f19d..f3ecca2f94 100644--- a/packages/backend/server/src/plugins/copilot/index.ts+++ b/packages/backend/server/src/plugins/copilot/index.ts@@ -1,6 +1,6 @@import { ServerFeature } from '../../core/config';-import { FeatureManagementService, FeatureService } from '../../core/features';-import { QuotaService } from '../../core/quota';+import { FeatureModule } from '../../core/features';+import { QuotaModule } from '../../core/quota';import { PermissionService } from '../../core/workspaces/permission';import { Plugin } from '../registry';import { CopilotController } from './controller';@@ -15,23 +15,23 @@ import {} from './providers';import { CopilotResolver, UserCopilotResolver } from './resolver';import { ChatSessionService } from './session';+import { CopilotStorage } from './storage';registerCopilotProvider(FalProvider);registerCopilotProvider(OpenAIProvider);@Plugin({name: 'copilot',+ imports: [FeatureModule, QuotaModule],providers: [PermissionService,- FeatureService,- FeatureManagementService,- QuotaService,ChatSessionService,CopilotResolver,ChatMessageCache,UserCopilotResolver,PromptService,CopilotProviderService,+ CopilotStorage,],controllers: [CopilotController],contributesTo: ServerFeature.Copilot,diff --git a/packages/backend/server/src/plugins/copilot/resolver.ts b/packages/backend/server/src/plugins/copilot/resolver.tsindex 18a774d6c2..f277c471c0 100644--- a/packages/backend/server/src/plugins/copilot/resolver.ts+++ b/packages/backend/server/src/plugins/copilot/resolver.ts@@ -1,4 +1,4 @@-import { Logger } from '@nestjs/common';+import { BadRequestException, Logger } from '@nestjs/common';import {Args,Field,@@ -12,12 +12,18 @@ import {Resolver,} from '@nestjs/graphql';import { GraphQLJSON, SafeIntResolver } from 'graphql-scalars';+import GraphQLUpload from 'graphql-upload/GraphQLUpload.mjs';import { CurrentUser } from '../../core/auth';import { UserType } from '../../core/user';import { PermissionService } from '../../core/workspaces/permission';-import { MutexService, TooManyRequestsException } from '../../fundamentals';+import {+ FileUpload,+ MutexService,+ TooManyRequestsException,+} from '../../fundamentals';import { ChatSessionService } from './session';+import { CopilotStorage } from './storage';import {AvailableModels,type ChatHistory,@@ -28,7 +34,7 @@ import {registerEnumType(AvailableModels, { name: 'CopilotModel' });-const COPILOT_LOCKER = 'copilot';+export const COPILOT_LOCKER = 'copilot';// ================== Input Types ==================@@ -57,6 +63,9 @@ class CreateChatMessageInput implements Omit<SubmittedMessage, 'content'> {@Field(() => [String], { nullable: true })attachments!: string[] | undefined;+ @Field(() => [GraphQLUpload], { nullable: true })+ blobs!: FileUpload[] | undefined;+@Field(() => GraphQLJSON, { nullable: true })params!: Record<string, string> | undefined;}@@ -140,7 +149,8 @@ export class CopilotResolver {constructor(private readonly permissions: PermissionService,private readonly mutex: MutexService,- private readonly chatSession: ChatSessionService+ private readonly chatSession: ChatSessionService,+ private readonly storage: CopilotStorage) {}@ResolveField(() => CopilotQuotaType, {@@ -260,6 +270,25 @@ export class CopilotResolver {if (!lock) {return new TooManyRequestsException('Server is busy');}+ const session = await this.chatSession.get(options.sessionId);+ if (!session) return new BadRequestException('Session not found');++ if (options.blobs) {+ options.attachments = options.attachments || [];+ const { workspaceId } = session.config;++ for (const blob of options.blobs) {+ const uploaded = await this.storage.handleUpload(user.id, blob);+ const link = await this.storage.put(+ user.id,+ workspaceId,+ uploaded.filename,+ uploaded.buffer+ );+ options.attachments.push(link);+ }+ }+try {return await this.chatSession.createMessage(options);} catch (e: any) {diff --git a/packages/backend/server/src/plugins/copilot/session.ts b/packages/backend/server/src/plugins/copilot/session.tsindex 90014b3b75..93a887f50d 100644--- a/packages/backend/server/src/plugins/copilot/session.ts+++ b/packages/backend/server/src/plugins/copilot/session.ts@@ -35,6 +35,18 @@ export class ChatSession implements AsyncDisposable {return this.state.prompt.model;}+ get config() {+ const {+ sessionId,+ userId,+ workspaceId,+ docId,+ prompt: { name: promptName },+ } = this.state;++ return { sessionId, userId, workspaceId, docId, promptName };+ }+push(message: ChatMessage) {if (this.state.prompt.action &&diff --git a/packages/backend/server/src/plugins/copilot/storage.ts b/packages/backend/server/src/plugins/copilot/storage.tsnew file mode 100644index 0000000000..ed49b63c64--- /dev/null+++ b/packages/backend/server/src/plugins/copilot/storage.ts@@ -0,0 +1,91 @@+import { createHash } from 'node:crypto';++import { Injectable, PayloadTooLargeException } from '@nestjs/common';++import { QuotaManagementService } from '../../core/quota';+import {+ type BlobInputType,+ Config,+ type FileUpload,+ type StorageProvider,+ StorageProviderFactory,+} from '../../fundamentals';++@Injectable()+export class CopilotStorage {+ public readonly provider: StorageProvider;++ constructor(+ private readonly config: Config,+ private readonly storageFactory: StorageProviderFactory,+ private readonly quota: QuotaManagementService+ ) {+ this.provider = this.storageFactory.create('copilot');+ }++ async put(+ userId: string,+ workspaceId: string,+ key: string,+ blob: BlobInputType+ ) {+ const name = `${userId}/${workspaceId}/${key}`;+ await this.provider.put(name, blob);+ return `${this.config.baseUrl}/api/copilot/blob/${name}`;+ }++ async get(userId: string, workspaceId: string, key: string) {+ return this.provider.get(`${userId}/${workspaceId}/${key}`);+ }++ async delete(userId: string, workspaceId: string, key: string) {+ return this.provider.delete(`${userId}/${workspaceId}/${key}`);+ }++ async handleUpload(userId: string, blob: FileUpload) {+ const checkExceeded = await this.quota.getQuotaCalculator(userId);++ if (checkExceeded(0)) {+ throw new PayloadTooLargeException(+ 'Storage or blob size limit exceeded.'+ );+ }+ const buffer = await new Promise<Buffer>((resolve, reject) => {+ const stream = blob.createReadStream();+ const chunks: Uint8Array[] = [];+ stream.on('data', chunk => {+ chunks.push(chunk);++ // check size after receive each chunk to avoid unnecessary memory usage+ const bufferSize = chunks.reduce((acc, cur) => acc + cur.length, 0);+ if (checkExceeded(bufferSize)) {+ reject(+ new PayloadTooLargeException('Storage or blob size limit exceeded.')+ );+ }+ });+ stream.on('error', reject);+ stream.on('end', () => {+ const buffer = Buffer.concat(chunks);++ if (checkExceeded(buffer.length)) {+ reject(new PayloadTooLargeException('Storage limit exceeded.'));+ } else {+ resolve(buffer);+ }+ });+ });++ return {+ buffer,+ filename: blob.filename,+ };+ }++ async handleRemoteLink(userId: string, workspaceId: string, link: string) {+ const response = await fetch(link);+ const buffer = new Uint8Array(await response.arrayBuffer());+ const filename = createHash('sha256').update(buffer).digest('base64url');+ return this.put(userId, workspaceId, filename, Buffer.from(buffer));+ }+}diff --git a/packages/backend/server/src/schema.gql b/packages/backend/server/src/schema.gqlindex 7915e8b1f8..da7ee6499c 100644--- a/packages/backend/server/src/schema.gql+++ b/packages/backend/server/src/schema.gql@@ -40,6 +40,7 @@ type CopilotQuota {input CreateChatMessageInput {attachments: [String!]+ blobs: [Upload!]content: Stringparams: JSONsessionId: String!diff --git a/packages/frontend/graphql/src/schema.ts b/packages/frontend/graphql/src/schema.tsindex 23e8f5f0a2..3d490692db 100644--- a/packages/frontend/graphql/src/schema.ts+++ b/packages/frontend/graphql/src/schema.ts@@ -38,6 +38,7 @@ export interface Scalars {export interface CreateChatMessageInput {attachments: InputMaybe<Array<Scalars['String']['input']>>;+ blobs: InputMaybe<Array<Scalars['Upload']['input']>>;content: InputMaybe<Scalars['String']['input']>;params: InputMaybe<Scalars['JSON']['input']>;sessionId: Scalars['String']['input'];