Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Store all uploads in a special table in server
Hakan Shehu81dd328d4b20
diff --git a/apps/desktop/src/main/services/workspaces/file-service.ts b/apps/desktop/src/main/services/workspaces/file-service.tsindex f01590bb..82cab682 100644--- a/apps/desktop/src/main/services/workspaces/file-service.ts+++ b/apps/desktop/src/main/services/workspaces/file-service.ts@@ -393,7 +393,9 @@ export class FileService {await this.workspace.account.client.put<CompleteUploadOutput>(`/v1/workspaces/${this.workspace.id}/files/${file.id}`,- {}+ {+ uploadId: data.uploadId,+ });const finalFileState = await this.workspace.databasediff --git a/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts b/apps/server/src/api/client/controllers/workspaces/files/file-download-get.tsindex 0b77d848..3fda2b8a 100644--- a/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts+++ b/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts@@ -6,20 +6,16 @@ import {extractNodeRole,FileStatus,} from '@colanode/core';-import { GetObjectCommand } from '@aws-sdk/client-s3';-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';import { fetchNodeTree, mapNode } from '@/lib/nodes';-import { fileS3 } from '@/data/storage';import { ResponseBuilder } from '@/lib/response-builder';-import { configuration } from '@/lib/configuration';-import { buildFilePath } from '@/lib/files';+import { database } from '@/data/database';+import { buildDownloadUrl } from '@/lib/files';export const fileDownloadGetHandler = async (req: Request,res: Response): Promise<void> => {- const workspaceId = req.params.workspaceId as string;const fileId = req.params.fileId as string;const tree = await fetchNodeTree(fileId);@@ -61,17 +57,20 @@ export const fileDownloadGetHandler = async (});}- //generate presigned url for download- const path = buildFilePath(workspaceId, file.id, file.attributes);- const command = new GetObjectCommand({- Bucket: configuration.fileS3.bucketName,- Key: path,- });+ const upload = await database+ .selectFrom('uploads')+ .selectAll()+ .where('file_id', '=', fileId)+ .executeTakeFirst();- const presignedUrl = await getSignedUrl(fileS3, command, {- expiresIn: 60 * 60 * 4, // 4 hours- });+ if (!upload || !upload.uploaded_at) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileUploadNotFound,+ message: 'File upload not found.',+ });+ }+ const presignedUrl = await buildDownloadUrl(upload.path);const output: CreateDownloadOutput = {url: presignedUrl,};diff --git a/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts b/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.tsindex d0055ee5..7bea7c6a 100644--- a/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts+++ b/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts@@ -1,13 +1,10 @@import { Request, Response } from 'express';-import { HeadObjectCommand } from '@aws-sdk/client-s3';import { FileStatus, ApiErrorCode } from '@colanode/core';import { database } from '@/data/database';-import { fileS3 } from '@/data/storage';import { ResponseBuilder } from '@/lib/response-builder';-import { configuration } from '@/lib/configuration';import { mapNode, updateNode } from '@/lib/nodes';-import { buildFilePath } from '@/lib/files';+import { fetchFileMetadata } from '@/lib/files';export const fileUploadCompleteHandler = async (req: Request,@@ -15,6 +12,7 @@ export const fileUploadCompleteHandler = async (): Promise<void> => {const workspaceId = req.params.workspaceId as string;const fileId = req.params.fileId as string;+ const uploadId = req.body.uploadId as string;const node = await database.selectFrom('nodes')@@ -64,35 +62,46 @@ export const fileUploadCompleteHandler = async (});}- const path = buildFilePath(workspaceId, file.id, file.attributes);- // check if the file exists in the bucket- const command = new HeadObjectCommand({- Bucket: configuration.fileS3.bucketName,- Key: path,- });+ const upload = await database+ .selectFrom('uploads')+ .selectAll()+ .where('file_id', '=', fileId)+ .executeTakeFirst();- try {- const headObject = await fileS3.send(command);-- // Verify file size matches expected size- if (headObject.ContentLength !== file.attributes.size) {- return ResponseBuilder.badRequest(res, {- code: ApiErrorCode.FileSizeMismatch,- message: 'Uploaded file size does not match expected size',- });- }-- // Verify mime type matches expected type- if (headObject.ContentType !== file.attributes.mimeType) {- return ResponseBuilder.badRequest(res, {- code: ApiErrorCode.FileMimeTypeMismatch,- message: 'Uploaded file type does not match expected type',- });- }- } catch {+ if (!upload) {return ResponseBuilder.badRequest(res, {- code: ApiErrorCode.FileError,- message: 'File upload verification failed',+ code: ApiErrorCode.FileUploadNotFound,+ message: 'Upload not found.',+ });+ }++ if (upload.file_id !== fileId || upload.upload_id !== uploadId) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileUploadNotFound,+ message: 'Upload not found.',+ });+ }++ const path = upload.path;+ const metadata = await fetchFileMetadata(path);+ if (metadata === null) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileNotFound,+ message: 'File not found.',+ });+ }++ if (metadata.size !== file.attributes.size) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileSizeMismatch,+ message: 'Uploaded file size does not match expected size',+ });+ }++ if (metadata.mimeType !== file.attributes.mimeType) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileMimeTypeMismatch,+ message: 'Uploaded file type does not match expected type',});}@@ -117,6 +126,14 @@ export const fileUploadCompleteHandler = async (});}+ await database+ .updateTable('uploads')+ .set({+ uploaded_at: new Date(),+ })+ .where('file_id', '=', fileId)+ .execute();+return ResponseBuilder.success(res, {success: true,});diff --git a/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts b/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.tsindex 862d32cc..0f6daefa 100644--- a/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts+++ b/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts@@ -3,23 +3,22 @@ import {CreateUploadInput,CreateUploadOutput,ApiErrorCode,+ generateId,+ IdType,} from '@colanode/core';-import { PutObjectCommand } from '@aws-sdk/client-s3';-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';import { database } from '@/data/database';-import { fileS3 } from '@/data/storage';import { ResponseBuilder } from '@/lib/response-builder';-import { configuration } from '@/lib/configuration';import { mapNode } from '@/lib/nodes';-import { buildFilePath } from '@/lib/files';+import { buildFilePath, buildUploadUrl } from '@/lib/files';+import { SelectUser } from '@/data/schema';export const fileUploadInitHandler = async (req: Request,res: Response): Promise<void> => {- const workspaceId = req.params.workspaceId as string;const input = req.body as CreateUploadInput;+ const user = res.locals.user as SelectUser;const node = await database.selectFrom('nodes')@@ -34,7 +33,7 @@ export const fileUploadInitHandler = async (});}- if (node.created_by !== res.locals.user.id) {+ if (node.created_by !== user.id) {return ResponseBuilder.forbidden(res, {code: ApiErrorCode.FileOwnerMismatch,message: 'You do not have access to this file.',@@ -50,21 +49,80 @@ export const fileUploadInitHandler = async (});}- const path = buildFilePath(workspaceId, input.fileId, file.attributes);- const command = new PutObjectCommand({- Bucket: configuration.fileS3.bucketName,- Key: path,- ContentLength: file.attributes.size,- ContentType: file.attributes.mimeType,- });+ const upload = await database+ .selectFrom('uploads')+ .selectAll()+ .where('file_id', '=', input.fileId)+ .executeTakeFirst();++ if (upload && upload.uploaded_at) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileAlreadyUploaded,+ message: 'This file is already uploaded.',+ });+ }++ const storageUsedRow = await database+ .selectFrom('uploads')+ .select(({ fn }) => [fn.sum('size').as('storage_used')])+ .where('created_by', '=', res.locals.user.id)+ .executeTakeFirst();++ const storageUsed = BigInt(storageUsedRow?.storage_used ?? 0);+ const storageLimit = user.storage_limit;++ if (storageUsed >= storageLimit) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileUploadInitFailed,+ message: 'You have reached the maximum storage limit.',+ });+ }++ const path = buildFilePath(node.workspace_id, input.fileId, file.attributes);+ const uploadId = generateId(IdType.Upload);+ const upsertedUpload = await database+ .insertInto('uploads')+ .values({+ file_id: input.fileId,+ upload_id: uploadId,+ workspace_id: node.workspace_id,+ root_id: node.id,+ mime_type: file.attributes.mimeType,+ size: file.attributes.size,+ path: path,+ version_id: file.attributes.version,+ created_at: new Date(),+ created_by: res.locals.user.id,+ })+ .onConflict((b) =>+ b.columns(['file_id']).doUpdateSet({+ upload_id: uploadId,+ mime_type: file.attributes.mimeType,+ size: file.attributes.size,+ path: path,+ version_id: file.attributes.version,+ created_at: new Date(),+ created_by: res.locals.user.id,+ })+ )+ .executeTakeFirst();++ if (!upsertedUpload) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileUploadInitFailed,+ message: 'Failed to initialize file upload.',+ });+ }- const expiresIn = 60 * 60 * 4; // 4 hours- const presignedUrl = await getSignedUrl(fileS3, command, {- expiresIn,- });+ const presignedUrl = await buildUploadUrl(+ path,+ file.attributes.size,+ file.attributes.mimeType+ );const output: CreateUploadOutput = {url: presignedUrl,+ uploadId: uploadId,};return ResponseBuilder.success(res, output);diff --git a/apps/server/src/data/migrations/00014-create-uploads-table.ts b/apps/server/src/data/migrations/00014-create-uploads-table.tsnew file mode 100644index 00000000..40274962--- /dev/null+++ b/apps/server/src/data/migrations/00014-create-uploads-table.ts@@ -0,0 +1,29 @@+import { Migration } from 'kysely';++export const createUploadsTable: Migration = {+ up: async (db) => {+ await db.schema+ .createTable('uploads')+ .addColumn('file_id', 'varchar(30)', (col) => col.notNull().primaryKey())+ .addColumn('upload_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('root_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('mime_type', 'varchar(255)', (col) => col.notNull())+ .addColumn('size', 'bigint', (col) => col.notNull())+ .addColumn('path', 'varchar(255)', (col) => col.notNull())+ .addColumn('version_id', 'varchar(30)', (col) => col.notNull())+ .addColumn('created_at', 'timestamptz', (col) => col.notNull())+ .addColumn('created_by', 'varchar(30)', (col) => col.notNull())+ .addColumn('uploaded_at', 'timestamptz')+ .execute();++ await db.schema+ .createIndex('uploads_created_by_idx')+ .on('uploads')+ .columns(['created_by'])+ .execute();+ },+ down: async (db) => {+ await db.schema.dropTable('uploads').execute();+ },+};diff --git a/apps/server/src/data/migrations/index.ts b/apps/server/src/data/migrations/index.tsindex 01840b7c..46f2281d 100644--- a/apps/server/src/data/migrations/index.ts+++ b/apps/server/src/data/migrations/index.ts@@ -13,6 +13,7 @@ import { createNodePathsTable } from './00010-create-node-paths-table';import { createCollaborationsTable } from './00011-create-collaborations-table';import { createDocumentsTable } from './00012-create-documents-table';import { createDocumentUpdatesTable } from './00013-create-document-updates-table';+import { createUploadsTable } from './00014-create-uploads-table';export const databaseMigrations: Record<string, Migration> = {'00001_create_accounts_table': createAccountsTable,@@ -28,4 +29,5 @@ export const databaseMigrations: Record<string, Migration> = {'00011_create_collaborations_table': createCollaborationsTable,'00012_create_documents_table': createDocumentsTable,'00013_create_document_updates_table': createDocumentUpdatesTable,+ '00014_create_uploads_table': createUploadsTable,};diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.tsindex 53660865..1eb86916 100644--- a/apps/server/src/data/schema.ts+++ b/apps/server/src/data/schema.ts@@ -240,6 +240,24 @@ export type SelectDocumentUpdate = Selectable<DocumentUpdateTable>;export type CreateDocumentUpdate = Insertable<DocumentUpdateTable>;export type UpdateDocumentUpdate = Updateable<DocumentUpdateTable>;+interface UploadTable {+ file_id: ColumnType<string, string, never>;+ upload_id: ColumnType<string, string, string>;+ workspace_id: ColumnType<string, string, never>;+ root_id: ColumnType<string, string, never>;+ mime_type: ColumnType<string, string, string>;+ size: ColumnType<number, number, number>;+ path: ColumnType<string, string, string>;+ version_id: ColumnType<string, string, string>;+ created_at: ColumnType<Date, Date, Date>;+ created_by: ColumnType<string, string, string>;+ uploaded_at: ColumnType<Date | null, Date | null, Date | null>;+}++export type SelectUpload = Selectable<UploadTable>;+export type CreateUpload = Insertable<UploadTable>;+export type UpdateUpload = Updateable<UploadTable>;+export interface DatabaseSchema {accounts: AccountTable;devices: DeviceTable;@@ -254,4 +272,5 @@ export interface DatabaseSchema {collaborations: CollaborationTable;documents: DocumentTable;document_updates: DocumentUpdateTable;+ uploads: UploadTable;}diff --git a/apps/server/src/jobs/clean-node-data.ts b/apps/server/src/jobs/clean-node-data.tsindex 2138bdf6..fdcd2e52 100644--- a/apps/server/src/jobs/clean-node-data.ts+++ b/apps/server/src/jobs/clean-node-data.ts@@ -4,6 +4,7 @@ import { database } from '@/data/database';import { JobHandler } from '@/types/jobs';import { eventBus } from '@/lib/event-bus';import { CreateNodeTombstone } from '@/data/schema';+import { deleteFile } from '@/lib/files';const BATCH_SIZE = 100;@@ -94,6 +95,23 @@ const deleteChildren = async (parentIds: string[], userId: string) => {.execute();});+ const uploads = await database+ .selectFrom('uploads')+ .selectAll()+ .where('file_id', 'in', nodeIds)+ .execute();++ if (uploads.length > 0) {+ for (const upload of uploads) {+ await deleteFile(upload.path);+ }++ await database+ .deleteFrom('uploads')+ .where('file_id', 'in', nodeIds)+ .execute();+ }+for (const node of descendants) {eventBus.publish({type: 'node_deleted',@@ -114,70 +132,3 @@ const deleteChildren = async (parentIds: string[], userId: string) => {return deletedNodeIds;};--// const deleteFiles = async (entryIds: string[], userId: string) => {-// let hasMore = true;-// while (hasMore) {-// try {-// const files = await database-// .selectFrom('files')-// .selectAll()-// .where('entry_id', 'in', entryIds)-// .orderBy('id', 'asc')-// .limit(BATCH_SIZE)-// .execute();--// if (files.length === 0) {-// hasMore = false;-// break;-// }--// const fileIds: string[] = files.map((m) => m.id);-// const fileTombstonesToCreate: CreateFileTombstone[] = files.map(-// (file) => ({-// id: file.id,-// root_id: file.root_id,-// workspace_id: file.workspace_id,-// deleted_at: new Date(),-// deleted_by: userId,-// })-// );--// await database.transaction().execute(async (trx) => {-// await trx.deleteFrom('files').where('id', 'in', fileIds).execute();--// await trx-// .deleteFrom('file_interactions')-// .where('file_id', 'in', fileIds)-// .execute();--// await trx-// .insertInto('file_tombstones')-// .values(fileTombstonesToCreate)-// .execute();-// });--// for (const file of files) {-// eventBus.publish({-// type: 'file_deleted',-// fileId: file.id,-// rootId: file.root_id,-// workspaceId: file.workspace_id,-// });--// const path = `files/${file.workspace_id}/${file.id}${file.extension}`;-// const command = new DeleteObjectCommand({-// Bucket: configuration.fileS3.bucketName,-// Key: path,-// });--// await fileS3.send(command);-// }--// hasMore = files.length === BATCH_SIZE;-// } catch (error) {-// debug(`Error deleting files for ${entryIds}: ${error}`);-// hasMore = false;-// }-// }-// };diff --git a/apps/server/src/lib/files.ts b/apps/server/src/lib/files.tsindex 5dfa7e2b..2c971542 100644--- a/apps/server/src/lib/files.ts+++ b/apps/server/src/lib/files.ts@@ -1,4 +1,14 @@+import {+ PutObjectCommand,+ DeleteObjectCommand,+ GetObjectCommand,+ HeadObjectCommand,+} from '@aws-sdk/client-s3';import { FileAttributes } from '@colanode/core';+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';++import { configuration } from '@/lib/configuration';+import { fileS3 } from '@/data/storage';export const buildFilePath = (workspaceId: string,@@ -7,3 +17,62 @@ export const buildFilePath = () => {return `files/${workspaceId}/${fileId}_${fileAttributes.version}${fileAttributes.extension}`;};++export const buildUploadUrl = async (+ path: string,+ size: number,+ mimeType: string+) => {+ const command = new PutObjectCommand({+ Bucket: configuration.fileS3.bucketName,+ Key: path,+ ContentLength: size,+ ContentType: mimeType,+ });++ const expiresIn = 60 * 60 * 4; // 4 hours+ const presignedUrl = await getSignedUrl(fileS3, command, {+ expiresIn,+ });++ return presignedUrl;+};++export const buildDownloadUrl = async (path: string) => {+ const command = new GetObjectCommand({+ Bucket: configuration.fileS3.bucketName,+ Key: path,+ });++ const presignedUrl = await getSignedUrl(fileS3, command, {+ expiresIn: 60 * 60 * 4, // 4 hours+ });++ return presignedUrl;+};++export const fetchFileMetadata = async (path: string) => {+ const command = new HeadObjectCommand({+ Bucket: configuration.fileS3.bucketName,+ Key: path,+ });++ try {+ const headObject = await fileS3.send(command);+ return {+ size: headObject.ContentLength,+ mimeType: headObject.ContentType,+ };+ } catch {+ return null;+ }+};++export const deleteFile = async (path: string) => {+ const command = new DeleteObjectCommand({+ Bucket: configuration.fileS3.bucketName,+ Key: path,+ });++ await fileS3.send(command);+};diff --git a/apps/server/src/lib/nodes.ts b/apps/server/src/lib/nodes.tsindex c3f726e0..13242291 100644--- a/apps/server/src/lib/nodes.ts+++ b/apps/server/src/lib/nodes.ts@@ -37,6 +37,7 @@ import {checkCollaboratorChanges,} from '@/lib/collaborations';import { jobService } from '@/services/job-service';+import { deleteFile } from '@/lib/files';const debug = createDebugger('server:lib:nodes');@@ -722,6 +723,23 @@ export const deleteNode = async (};});+ if (deletedNode.type === 'file') {+ const upload = await database+ .selectFrom('uploads')+ .selectAll()+ .where('file_id', '=', input.nodeId)+ .executeTakeFirst();++ if (upload) {+ await deleteFile(upload.path);++ await database+ .deleteFrom('uploads')+ .where('file_id', '=', input.nodeId)+ .execute();+ }+ }+eventBus.publish({type: 'node_deleted',nodeId: input.nodeId,diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.tsindex 0b918b8e..d331122d 100644--- a/packages/core/src/types/api.ts+++ b/packages/core/src/types/api.ts@@ -28,11 +28,14 @@ export enum ApiErrorCode {FileNoAccess = 'file_no_access',FileNotReady = 'file_not_ready',FileOwnerMismatch = 'file_owner_mismatch',+ FileAlreadyUploaded = 'file_already_uploaded',+ FileUploadInitFailed = 'file_upload_init_failed',WorkspaceMismatch = 'workspace_mismatch',FileError = 'file_error',FileSizeMismatch = 'file_size_mismatch',FileMimeTypeMismatch = 'file_mime_type_mismatch',FileUploadCompleteFailed = 'file_upload_complete_failed',+ FileUploadNotFound = 'file_upload_not_found',UserEmailRequired = 'user_email_required',UserInviteNoAccess = 'user_invite_no_access',UserUpdateNoAccess = 'user_update_no_access',diff --git a/packages/core/src/types/files.ts b/packages/core/src/types/files.tsindex a1a1ebdd..ca116d9d 100644--- a/packages/core/src/types/files.ts+++ b/packages/core/src/types/files.ts@@ -4,6 +4,7 @@ export type CreateUploadInput = {export type CreateUploadOutput = {url: string;+ uploadId: string;};export type CreateDownloadOutput = {