Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Store all uploads in a special table in server

Hakan Shehu81dd328d4b20
diff --git a/apps/desktop/src/main/services/workspaces/file-service.ts b/apps/desktop/src/main/services/workspaces/file-service.ts
index f01590bb..82cab682 100644
--- a/apps/desktop/src/main/services/workspaces/file-service.ts
+++ b/apps/desktop/src/main/services/workspaces/file-service.ts
@@ -393,7 +393,9 @@ export class FileService {
await this.workspace.account.client.put<CompleteUploadOutput>(
`/v1/workspaces/${this.workspace.id}/files/${file.id}`,
- {}
+ {
+ uploadId: data.uploadId,
+ }
);
const finalFileState = await this.workspace.database
diff --git a/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts b/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts
index 0b77d848..3fda2b8a 100644
--- a/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts
+++ b/apps/server/src/api/client/controllers/workspaces/files/file-download-get.ts
@@ -6,20 +6,16 @@ import {
extractNodeRole,
FileStatus,
} from '@colanode/core';
-import { GetObjectCommand } from '@aws-sdk/client-s3';
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { fetchNodeTree, mapNode } from '@/lib/nodes';
-import { fileS3 } from '@/data/storage';
import { ResponseBuilder } from '@/lib/response-builder';
-import { configuration } from '@/lib/configuration';
-import { buildFilePath } from '@/lib/files';
+import { database } from '@/data/database';
+import { buildDownloadUrl } from '@/lib/files';
export const fileDownloadGetHandler = async (
req: Request,
res: Response
): Promise<void> => {
- const workspaceId = req.params.workspaceId as string;
const fileId = req.params.fileId as string;
const tree = await fetchNodeTree(fileId);
@@ -61,17 +57,20 @@ export const fileDownloadGetHandler = async (
});
}
- //generate presigned url for download
- const path = buildFilePath(workspaceId, file.id, file.attributes);
- const command = new GetObjectCommand({
- Bucket: configuration.fileS3.bucketName,
- Key: path,
- });
+ const upload = await database
+ .selectFrom('uploads')
+ .selectAll()
+ .where('file_id', '=', fileId)
+ .executeTakeFirst();
- const presignedUrl = await getSignedUrl(fileS3, command, {
- expiresIn: 60 * 60 * 4, // 4 hours
- });
+ if (!upload || !upload.uploaded_at) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileUploadNotFound,
+ message: 'File upload not found.',
+ });
+ }
+ const presignedUrl = await buildDownloadUrl(upload.path);
const output: CreateDownloadOutput = {
url: presignedUrl,
};
diff --git a/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts b/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts
index d0055ee5..7bea7c6a 100644
--- a/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts
+++ b/apps/server/src/api/client/controllers/workspaces/files/file-upload-complete.ts
@@ -1,13 +1,10 @@
import { Request, Response } from 'express';
-import { HeadObjectCommand } from '@aws-sdk/client-s3';
import { FileStatus, ApiErrorCode } from '@colanode/core';
import { database } from '@/data/database';
-import { fileS3 } from '@/data/storage';
import { ResponseBuilder } from '@/lib/response-builder';
-import { configuration } from '@/lib/configuration';
import { mapNode, updateNode } from '@/lib/nodes';
-import { buildFilePath } from '@/lib/files';
+import { fetchFileMetadata } from '@/lib/files';
export const fileUploadCompleteHandler = async (
req: Request,
@@ -15,6 +12,7 @@ export const fileUploadCompleteHandler = async (
): Promise<void> => {
const workspaceId = req.params.workspaceId as string;
const fileId = req.params.fileId as string;
+ const uploadId = req.body.uploadId as string;
const node = await database
.selectFrom('nodes')
@@ -64,35 +62,46 @@ export const fileUploadCompleteHandler = async (
});
}
- const path = buildFilePath(workspaceId, file.id, file.attributes);
- // check if the file exists in the bucket
- const command = new HeadObjectCommand({
- Bucket: configuration.fileS3.bucketName,
- Key: path,
- });
+ const upload = await database
+ .selectFrom('uploads')
+ .selectAll()
+ .where('file_id', '=', fileId)
+ .executeTakeFirst();
- try {
- const headObject = await fileS3.send(command);
-
- // Verify file size matches expected size
- if (headObject.ContentLength !== file.attributes.size) {
- return ResponseBuilder.badRequest(res, {
- code: ApiErrorCode.FileSizeMismatch,
- message: 'Uploaded file size does not match expected size',
- });
- }
-
- // Verify mime type matches expected type
- if (headObject.ContentType !== file.attributes.mimeType) {
- return ResponseBuilder.badRequest(res, {
- code: ApiErrorCode.FileMimeTypeMismatch,
- message: 'Uploaded file type does not match expected type',
- });
- }
- } catch {
+ if (!upload) {
return ResponseBuilder.badRequest(res, {
- code: ApiErrorCode.FileError,
- message: 'File upload verification failed',
+ code: ApiErrorCode.FileUploadNotFound,
+ message: 'Upload not found.',
+ });
+ }
+
+ if (upload.file_id !== fileId || upload.upload_id !== uploadId) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileUploadNotFound,
+ message: 'Upload not found.',
+ });
+ }
+
+ const path = upload.path;
+ const metadata = await fetchFileMetadata(path);
+ if (metadata === null) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileNotFound,
+ message: 'File not found.',
+ });
+ }
+
+ if (metadata.size !== file.attributes.size) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileSizeMismatch,
+ message: 'Uploaded file size does not match expected size',
+ });
+ }
+
+ if (metadata.mimeType !== file.attributes.mimeType) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileMimeTypeMismatch,
+ message: 'Uploaded file type does not match expected type',
});
}
@@ -117,6 +126,14 @@ export const fileUploadCompleteHandler = async (
});
}
+ await database
+ .updateTable('uploads')
+ .set({
+ uploaded_at: new Date(),
+ })
+ .where('file_id', '=', fileId)
+ .execute();
+
return ResponseBuilder.success(res, {
success: true,
});
diff --git a/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts b/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts
index 862d32cc..0f6daefa 100644
--- a/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts
+++ b/apps/server/src/api/client/controllers/workspaces/files/file-upload-init.ts
@@ -3,23 +3,22 @@ import {
CreateUploadInput,
CreateUploadOutput,
ApiErrorCode,
+ generateId,
+ IdType,
} from '@colanode/core';
-import { PutObjectCommand } from '@aws-sdk/client-s3';
-import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
import { database } from '@/data/database';
-import { fileS3 } from '@/data/storage';
import { ResponseBuilder } from '@/lib/response-builder';
-import { configuration } from '@/lib/configuration';
import { mapNode } from '@/lib/nodes';
-import { buildFilePath } from '@/lib/files';
+import { buildFilePath, buildUploadUrl } from '@/lib/files';
+import { SelectUser } from '@/data/schema';
export const fileUploadInitHandler = async (
req: Request,
res: Response
): Promise<void> => {
- const workspaceId = req.params.workspaceId as string;
const input = req.body as CreateUploadInput;
+ const user = res.locals.user as SelectUser;
const node = await database
.selectFrom('nodes')
@@ -34,7 +33,7 @@ export const fileUploadInitHandler = async (
});
}
- if (node.created_by !== res.locals.user.id) {
+ if (node.created_by !== user.id) {
return ResponseBuilder.forbidden(res, {
code: ApiErrorCode.FileOwnerMismatch,
message: 'You do not have access to this file.',
@@ -50,21 +49,80 @@ export const fileUploadInitHandler = async (
});
}
- const path = buildFilePath(workspaceId, input.fileId, file.attributes);
- const command = new PutObjectCommand({
- Bucket: configuration.fileS3.bucketName,
- Key: path,
- ContentLength: file.attributes.size,
- ContentType: file.attributes.mimeType,
- });
+ const upload = await database
+ .selectFrom('uploads')
+ .selectAll()
+ .where('file_id', '=', input.fileId)
+ .executeTakeFirst();
+
+ if (upload && upload.uploaded_at) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileAlreadyUploaded,
+ message: 'This file is already uploaded.',
+ });
+ }
+
+ const storageUsedRow = await database
+ .selectFrom('uploads')
+ .select(({ fn }) => [fn.sum('size').as('storage_used')])
+ .where('created_by', '=', res.locals.user.id)
+ .executeTakeFirst();
+
+ const storageUsed = BigInt(storageUsedRow?.storage_used ?? 0);
+ const storageLimit = user.storage_limit;
+
+ if (storageUsed >= storageLimit) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileUploadInitFailed,
+ message: 'You have reached the maximum storage limit.',
+ });
+ }
+
+ const path = buildFilePath(node.workspace_id, input.fileId, file.attributes);
+ const uploadId = generateId(IdType.Upload);
+ const upsertedUpload = await database
+ .insertInto('uploads')
+ .values({
+ file_id: input.fileId,
+ upload_id: uploadId,
+ workspace_id: node.workspace_id,
+ root_id: node.id,
+ mime_type: file.attributes.mimeType,
+ size: file.attributes.size,
+ path: path,
+ version_id: file.attributes.version,
+ created_at: new Date(),
+ created_by: res.locals.user.id,
+ })
+ .onConflict((b) =>
+ b.columns(['file_id']).doUpdateSet({
+ upload_id: uploadId,
+ mime_type: file.attributes.mimeType,
+ size: file.attributes.size,
+ path: path,
+ version_id: file.attributes.version,
+ created_at: new Date(),
+ created_by: res.locals.user.id,
+ })
+ )
+ .executeTakeFirst();
+
+ if (!upsertedUpload) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileUploadInitFailed,
+ message: 'Failed to initialize file upload.',
+ });
+ }
- const expiresIn = 60 * 60 * 4; // 4 hours
- const presignedUrl = await getSignedUrl(fileS3, command, {
- expiresIn,
- });
+ const presignedUrl = await buildUploadUrl(
+ path,
+ file.attributes.size,
+ file.attributes.mimeType
+ );
const output: CreateUploadOutput = {
url: presignedUrl,
+ uploadId: uploadId,
};
return ResponseBuilder.success(res, output);
diff --git a/apps/server/src/data/migrations/00014-create-uploads-table.ts b/apps/server/src/data/migrations/00014-create-uploads-table.ts
new file mode 100644
index 00000000..40274962
--- /dev/null
+++ b/apps/server/src/data/migrations/00014-create-uploads-table.ts
@@ -0,0 +1,29 @@
+import { Migration } from 'kysely';
+
+export const createUploadsTable: Migration = {
+ up: async (db) => {
+ await db.schema
+ .createTable('uploads')
+ .addColumn('file_id', 'varchar(30)', (col) => col.notNull().primaryKey())
+ .addColumn('upload_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('workspace_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('root_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('mime_type', 'varchar(255)', (col) => col.notNull())
+ .addColumn('size', 'bigint', (col) => col.notNull())
+ .addColumn('path', 'varchar(255)', (col) => col.notNull())
+ .addColumn('version_id', 'varchar(30)', (col) => col.notNull())
+ .addColumn('created_at', 'timestamptz', (col) => col.notNull())
+ .addColumn('created_by', 'varchar(30)', (col) => col.notNull())
+ .addColumn('uploaded_at', 'timestamptz')
+ .execute();
+
+ await db.schema
+ .createIndex('uploads_created_by_idx')
+ .on('uploads')
+ .columns(['created_by'])
+ .execute();
+ },
+ down: async (db) => {
+ await db.schema.dropTable('uploads').execute();
+ },
+};
diff --git a/apps/server/src/data/migrations/index.ts b/apps/server/src/data/migrations/index.ts
index 01840b7c..46f2281d 100644
--- a/apps/server/src/data/migrations/index.ts
+++ b/apps/server/src/data/migrations/index.ts
@@ -13,6 +13,7 @@ import { createNodePathsTable } from './00010-create-node-paths-table';
import { createCollaborationsTable } from './00011-create-collaborations-table';
import { createDocumentsTable } from './00012-create-documents-table';
import { createDocumentUpdatesTable } from './00013-create-document-updates-table';
+import { createUploadsTable } from './00014-create-uploads-table';
export const databaseMigrations: Record<string, Migration> = {
'00001_create_accounts_table': createAccountsTable,
@@ -28,4 +29,5 @@ export const databaseMigrations: Record<string, Migration> = {
'00011_create_collaborations_table': createCollaborationsTable,
'00012_create_documents_table': createDocumentsTable,
'00013_create_document_updates_table': createDocumentUpdatesTable,
+ '00014_create_uploads_table': createUploadsTable,
};
diff --git a/apps/server/src/data/schema.ts b/apps/server/src/data/schema.ts
index 53660865..1eb86916 100644
--- a/apps/server/src/data/schema.ts
+++ b/apps/server/src/data/schema.ts
@@ -240,6 +240,24 @@ export type SelectDocumentUpdate = Selectable<DocumentUpdateTable>;
export type CreateDocumentUpdate = Insertable<DocumentUpdateTable>;
export type UpdateDocumentUpdate = Updateable<DocumentUpdateTable>;
+interface UploadTable {
+ file_id: ColumnType<string, string, never>;
+ upload_id: ColumnType<string, string, string>;
+ workspace_id: ColumnType<string, string, never>;
+ root_id: ColumnType<string, string, never>;
+ mime_type: ColumnType<string, string, string>;
+ size: ColumnType<number, number, number>;
+ path: ColumnType<string, string, string>;
+ version_id: ColumnType<string, string, string>;
+ created_at: ColumnType<Date, Date, Date>;
+ created_by: ColumnType<string, string, string>;
+ uploaded_at: ColumnType<Date | null, Date | null, Date | null>;
+}
+
+export type SelectUpload = Selectable<UploadTable>;
+export type CreateUpload = Insertable<UploadTable>;
+export type UpdateUpload = Updateable<UploadTable>;
+
export interface DatabaseSchema {
accounts: AccountTable;
devices: DeviceTable;
@@ -254,4 +272,5 @@ export interface DatabaseSchema {
collaborations: CollaborationTable;
documents: DocumentTable;
document_updates: DocumentUpdateTable;
+ uploads: UploadTable;
}
diff --git a/apps/server/src/jobs/clean-node-data.ts b/apps/server/src/jobs/clean-node-data.ts
index 2138bdf6..fdcd2e52 100644
--- a/apps/server/src/jobs/clean-node-data.ts
+++ b/apps/server/src/jobs/clean-node-data.ts
@@ -4,6 +4,7 @@ import { database } from '@/data/database';
import { JobHandler } from '@/types/jobs';
import { eventBus } from '@/lib/event-bus';
import { CreateNodeTombstone } from '@/data/schema';
+import { deleteFile } from '@/lib/files';
const BATCH_SIZE = 100;
@@ -94,6 +95,23 @@ const deleteChildren = async (parentIds: string[], userId: string) => {
.execute();
});
+ const uploads = await database
+ .selectFrom('uploads')
+ .selectAll()
+ .where('file_id', 'in', nodeIds)
+ .execute();
+
+ if (uploads.length > 0) {
+ for (const upload of uploads) {
+ await deleteFile(upload.path);
+ }
+
+ await database
+ .deleteFrom('uploads')
+ .where('file_id', 'in', nodeIds)
+ .execute();
+ }
+
for (const node of descendants) {
eventBus.publish({
type: 'node_deleted',
@@ -114,70 +132,3 @@ const deleteChildren = async (parentIds: string[], userId: string) => {
return deletedNodeIds;
};
-
-// const deleteFiles = async (entryIds: string[], userId: string) => {
-// let hasMore = true;
-// while (hasMore) {
-// try {
-// const files = await database
-// .selectFrom('files')
-// .selectAll()
-// .where('entry_id', 'in', entryIds)
-// .orderBy('id', 'asc')
-// .limit(BATCH_SIZE)
-// .execute();
-
-// if (files.length === 0) {
-// hasMore = false;
-// break;
-// }
-
-// const fileIds: string[] = files.map((m) => m.id);
-// const fileTombstonesToCreate: CreateFileTombstone[] = files.map(
-// (file) => ({
-// id: file.id,
-// root_id: file.root_id,
-// workspace_id: file.workspace_id,
-// deleted_at: new Date(),
-// deleted_by: userId,
-// })
-// );
-
-// await database.transaction().execute(async (trx) => {
-// await trx.deleteFrom('files').where('id', 'in', fileIds).execute();
-
-// await trx
-// .deleteFrom('file_interactions')
-// .where('file_id', 'in', fileIds)
-// .execute();
-
-// await trx
-// .insertInto('file_tombstones')
-// .values(fileTombstonesToCreate)
-// .execute();
-// });
-
-// for (const file of files) {
-// eventBus.publish({
-// type: 'file_deleted',
-// fileId: file.id,
-// rootId: file.root_id,
-// workspaceId: file.workspace_id,
-// });
-
-// const path = `files/${file.workspace_id}/${file.id}${file.extension}`;
-// const command = new DeleteObjectCommand({
-// Bucket: configuration.fileS3.bucketName,
-// Key: path,
-// });
-
-// await fileS3.send(command);
-// }
-
-// hasMore = files.length === BATCH_SIZE;
-// } catch (error) {
-// debug(`Error deleting files for ${entryIds}: ${error}`);
-// hasMore = false;
-// }
-// }
-// };
diff --git a/apps/server/src/lib/files.ts b/apps/server/src/lib/files.ts
index 5dfa7e2b..2c971542 100644
--- a/apps/server/src/lib/files.ts
+++ b/apps/server/src/lib/files.ts
@@ -1,4 +1,14 @@
+import {
+ PutObjectCommand,
+ DeleteObjectCommand,
+ GetObjectCommand,
+ HeadObjectCommand,
+} from '@aws-sdk/client-s3';
import { FileAttributes } from '@colanode/core';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+
+import { configuration } from '@/lib/configuration';
+import { fileS3 } from '@/data/storage';
export const buildFilePath = (
workspaceId: string,
@@ -7,3 +17,62 @@ export const buildFilePath = (
) => {
return `files/${workspaceId}/${fileId}_${fileAttributes.version}${fileAttributes.extension}`;
};
+
+export const buildUploadUrl = async (
+ path: string,
+ size: number,
+ mimeType: string
+) => {
+ const command = new PutObjectCommand({
+ Bucket: configuration.fileS3.bucketName,
+ Key: path,
+ ContentLength: size,
+ ContentType: mimeType,
+ });
+
+ const expiresIn = 60 * 60 * 4; // 4 hours
+ const presignedUrl = await getSignedUrl(fileS3, command, {
+ expiresIn,
+ });
+
+ return presignedUrl;
+};
+
+export const buildDownloadUrl = async (path: string) => {
+ const command = new GetObjectCommand({
+ Bucket: configuration.fileS3.bucketName,
+ Key: path,
+ });
+
+ const presignedUrl = await getSignedUrl(fileS3, command, {
+ expiresIn: 60 * 60 * 4, // 4 hours
+ });
+
+ return presignedUrl;
+};
+
+export const fetchFileMetadata = async (path: string) => {
+ const command = new HeadObjectCommand({
+ Bucket: configuration.fileS3.bucketName,
+ Key: path,
+ });
+
+ try {
+ const headObject = await fileS3.send(command);
+ return {
+ size: headObject.ContentLength,
+ mimeType: headObject.ContentType,
+ };
+ } catch {
+ return null;
+ }
+};
+
+export const deleteFile = async (path: string) => {
+ const command = new DeleteObjectCommand({
+ Bucket: configuration.fileS3.bucketName,
+ Key: path,
+ });
+
+ await fileS3.send(command);
+};
diff --git a/apps/server/src/lib/nodes.ts b/apps/server/src/lib/nodes.ts
index c3f726e0..13242291 100644
--- a/apps/server/src/lib/nodes.ts
+++ b/apps/server/src/lib/nodes.ts
@@ -37,6 +37,7 @@ import {
checkCollaboratorChanges,
} from '@/lib/collaborations';
import { jobService } from '@/services/job-service';
+import { deleteFile } from '@/lib/files';
const debug = createDebugger('server:lib:nodes');
@@ -722,6 +723,23 @@ export const deleteNode = async (
};
});
+ if (deletedNode.type === 'file') {
+ const upload = await database
+ .selectFrom('uploads')
+ .selectAll()
+ .where('file_id', '=', input.nodeId)
+ .executeTakeFirst();
+
+ if (upload) {
+ await deleteFile(upload.path);
+
+ await database
+ .deleteFrom('uploads')
+ .where('file_id', '=', input.nodeId)
+ .execute();
+ }
+ }
+
eventBus.publish({
type: 'node_deleted',
nodeId: input.nodeId,
diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts
index 0b918b8e..d331122d 100644
--- a/packages/core/src/types/api.ts
+++ b/packages/core/src/types/api.ts
@@ -28,11 +28,14 @@ export enum ApiErrorCode {
FileNoAccess = 'file_no_access',
FileNotReady = 'file_not_ready',
FileOwnerMismatch = 'file_owner_mismatch',
+ FileAlreadyUploaded = 'file_already_uploaded',
+ FileUploadInitFailed = 'file_upload_init_failed',
WorkspaceMismatch = 'workspace_mismatch',
FileError = 'file_error',
FileSizeMismatch = 'file_size_mismatch',
FileMimeTypeMismatch = 'file_mime_type_mismatch',
FileUploadCompleteFailed = 'file_upload_complete_failed',
+ FileUploadNotFound = 'file_upload_not_found',
UserEmailRequired = 'user_email_required',
UserInviteNoAccess = 'user_invite_no_access',
UserUpdateNoAccess = 'user_update_no_access',
diff --git a/packages/core/src/types/files.ts b/packages/core/src/types/files.ts
index a1a1ebdd..ca116d9d 100644
--- a/packages/core/src/types/files.ts
+++ b/packages/core/src/types/files.ts
@@ -4,6 +4,7 @@ export type CreateUploadInput = {
export type CreateUploadOutput = {
url: string;
+ uploadId: string;
};
export type CreateDownloadOutput = {