Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Improve file state handling
Hakan Shehu4fb71bcaad40
diff --git a/apps/desktop/src/main/data/workspace/migrations.ts b/apps/desktop/src/main/data/workspace/migrations.tsindex c576fde1..1b131b32 100644--- a/apps/desktop/src/main/data/workspace/migrations.ts+++ b/apps/desktop/src/main/data/workspace/migrations.ts@@ -238,10 +238,10 @@ const createFileStatesTable: Migration = {await db.schema.createTable('file_states').addColumn('file_id', 'text', (col) => col.notNull().primaryKey())- .addColumn('download_status', 'text', (col) => col.notNull())+ .addColumn('download_status', 'integer', (col) => col.notNull()).addColumn('download_progress', 'integer', (col) => col.notNull()).addColumn('download_retries', 'integer', (col) => col.notNull())- .addColumn('upload_status', 'text', (col) => col.notNull())+ .addColumn('upload_status', 'integer', (col) => col.notNull()).addColumn('upload_progress', 'integer', (col) => col.notNull()).addColumn('upload_retries', 'integer', (col) => col.notNull()).addColumn('created_at', 'text', (col) => col.notNull())diff --git a/apps/desktop/src/main/mutations/files/file-create.ts b/apps/desktop/src/main/mutations/files/file-create.tsindex 45d108cd..bd63dcf6 100644--- a/apps/desktop/src/main/mutations/files/file-create.ts+++ b/apps/desktop/src/main/mutations/files/file-create.ts@@ -23,6 +23,7 @@ import {mapFile,} from '@/main/utils';import { formatBytes } from '@/shared/lib/files';+import { DownloadStatus, UploadStatus } from '@/shared/types/files';export class FileCreateMutationHandlerimplements MutationHandler<FileCreateMutationInput>@@ -167,10 +168,10 @@ export class FileCreateMutationHandlerfile_id: fileId,created_at: new Date().toISOString(),download_progress: 100,- download_status: 'completed',+ download_status: DownloadStatus.Completed,download_retries: 0,upload_progress: 0,- upload_status: 'pending',+ upload_status: UploadStatus.Pending,upload_retries: 0,}).execute();@@ -211,10 +212,10 @@ export class FileCreateMutationHandlerfileState: {fileId: fileId,downloadProgress: 100,- downloadStatus: 'completed',+ downloadStatus: DownloadStatus.Completed,downloadRetries: 0,uploadProgress: 10,- uploadStatus: 'pending',+ uploadStatus: UploadStatus.Pending,uploadRetries: 0,createdAt: new Date().toISOString(),updatedAt: null,diff --git a/apps/desktop/src/main/mutations/files/file-download.ts b/apps/desktop/src/main/mutations/files/file-download.tsindex dba41abf..5f486c26 100644--- a/apps/desktop/src/main/mutations/files/file-download.ts+++ b/apps/desktop/src/main/mutations/files/file-download.ts@@ -1,3 +1,5 @@+import { FileStatus } from '@colanode/core';+import { databaseService } from '@/main/data/database-service';import { MutationHandler } from '@/main/types';import { mapFileState } from '@/main/utils';@@ -7,6 +9,7 @@ import {FileDownloadMutationInput,FileDownloadMutationOutput,} from '@/shared/mutations/files/file-download';+import { DownloadStatus, UploadStatus } from '@/shared/types/files';export class FileDownloadMutationHandlerimplements MutationHandler<FileDownloadMutationInput>@@ -31,6 +34,13 @@ export class FileDownloadMutationHandler);}+ if (file.status !== FileStatus.Ready) {+ throw new MutationError(+ MutationErrorCode.FileNotReady,+ 'The file you are trying to download is not uploaded by the author yet.'+ );+ }+const existingFileState = await workspaceDatabase.selectFrom('file_states').selectAll()@@ -39,7 +49,7 @@ export class FileDownloadMutationHandlerif (existingFileState &&- existingFileState.download_status === 'completed'+ existingFileState.download_status === DownloadStatus.Completed) {return {success: true,@@ -51,17 +61,17 @@ export class FileDownloadMutationHandler.returningAll().values({file_id: input.fileId,- download_status: 'pending',+ download_status: DownloadStatus.Pending,download_progress: 0,download_retries: 0,- upload_status: 'none',+ upload_status: UploadStatus.None,upload_progress: 0,upload_retries: 0,created_at: new Date().toISOString(),}).onConflict((oc) =>oc.doUpdateSet(() => ({- download_status: 'pending',+ download_status: DownloadStatus.Pending,download_progress: 0,download_retries: 0,updated_at: new Date().toISOString(),diff --git a/apps/desktop/src/main/mutations/messages/message-create.ts b/apps/desktop/src/main/mutations/messages/message-create.tsindex ce957908..73f18f1a 100644--- a/apps/desktop/src/main/mutations/messages/message-create.ts+++ b/apps/desktop/src/main/mutations/messages/message-create.ts@@ -36,6 +36,7 @@ import {mapMessage,} from '@/main/utils';import { formatBytes } from '@/shared/lib/files';+import { DownloadStatus, UploadStatus } from '@/shared/types/files';export class MessageCreateMutationHandlerimplements MutationHandler<MessageCreateMutationInput>@@ -170,11 +171,11 @@ export class MessageCreateMutationHandlerfileStates.push({file_id: fileId,- upload_status: 'pending',+ upload_status: UploadStatus.Pending,upload_progress: 0,upload_retries: 0,- download_status: 'pending',- download_progress: 0,+ download_status: DownloadStatus.Completed,+ download_progress: 100,download_retries: 0,created_at: createdAt,});diff --git a/apps/desktop/src/main/queries/files/file-get.ts b/apps/desktop/src/main/queries/files/file-get.tsindex b11baad3..9bba92bb 100644--- a/apps/desktop/src/main/queries/files/file-get.ts+++ b/apps/desktop/src/main/queries/files/file-get.ts@@ -3,7 +3,11 @@ import { ChangeCheckResult, QueryHandler } from '@/main/types';import { mapFile } from '@/main/utils';import { FileGetQueryInput } from '@/shared/queries/files/file-get';import { Event } from '@/shared/types/events';-import { FileWithState } from '@/shared/types/files';+import {+ DownloadStatus,+ FileWithState,+ UploadStatus,+} from '@/shared/types/files';export class FileGetQueryHandler implements QueryHandler<FileGetQueryInput> {public async handleQuery(@@ -49,9 +53,9 @@ export class FileGetQueryHandler implements QueryHandler<FileGetQueryInput> {result: {...event.file,downloadProgress: output?.downloadProgress ?? 0,- downloadStatus: output?.downloadStatus ?? 'none',+ downloadStatus: output?.downloadStatus ?? DownloadStatus.None,uploadProgress: output?.uploadProgress ?? 0,- uploadStatus: output?.uploadStatus ?? 'none',+ uploadStatus: output?.uploadStatus ?? UploadStatus.None,},};}@@ -160,9 +164,9 @@ export class FileGetQueryHandler implements QueryHandler<FileGetQueryInput> {const fileWithState: FileWithState = {...mapFile(file),downloadProgress: file.download_progress ?? 0,- downloadStatus: file.download_status ?? 'none',+ downloadStatus: file.download_status ?? DownloadStatus.None,uploadProgress: file.upload_progress ?? 0,- uploadStatus: file.upload_status ?? 'none',+ uploadStatus: file.upload_status ?? UploadStatus.None,};return fileWithState;diff --git a/apps/desktop/src/main/queries/files/file-list.ts b/apps/desktop/src/main/queries/files/file-list.tsindex 7d552283..d4b54491 100644--- a/apps/desktop/src/main/queries/files/file-list.ts+++ b/apps/desktop/src/main/queries/files/file-list.ts@@ -3,7 +3,11 @@ import { ChangeCheckResult, QueryHandler } from '@/main/types';import { mapFile } from '@/main/utils';import { FileListQueryInput } from '@/shared/queries/files/file-list';import { Event } from '@/shared/types/events';-import { FileWithState } from '@/shared/types/files';+import {+ DownloadStatus,+ FileWithState,+ UploadStatus,+} from '@/shared/types/files';export class FileListQueryHandler implements QueryHandler<FileListQueryInput> {public async handleQuery(@@ -171,9 +175,9 @@ export class FileListQueryHandler implements QueryHandler<FileListQueryInput> {const filesWithState: FileWithState[] = files.map((file) => ({...mapFile(file),downloadProgress: file.download_progress ?? 0,- downloadStatus: file.download_status ?? 'none',+ downloadStatus: file.download_status ?? DownloadStatus.None,uploadProgress: file.upload_progress ?? 0,- uploadStatus: file.upload_status ?? 'none',+ uploadStatus: file.upload_status ?? UploadStatus.None,}));return filesWithState;diff --git a/apps/desktop/src/main/services/file-service.ts b/apps/desktop/src/main/services/file-service.tsindex cbc8550f..25332d28 100644--- a/apps/desktop/src/main/services/file-service.ts+++ b/apps/desktop/src/main/services/file-service.ts@@ -27,7 +27,11 @@ import {} from '@/main/utils';import { eventBus } from '@/shared/lib/event-bus';import { httpClient } from '@/shared/lib/http-client';-import { FileMetadata } from '@/shared/types/files';+import {+ DownloadStatus,+ FileMetadata,+ UploadStatus,+} from '@/shared/types/files';class FileService {private readonly debug = createDebugger('service:file');@@ -139,7 +143,7 @@ class FileService {const uploads = await workspaceDatabase.selectFrom('file_states').selectAll()- .where('upload_status', '=', 'pending')+ .where('upload_status', '=', UploadStatus.Pending).execute();if (uploads.length === 0) {@@ -158,7 +162,7 @@ class FileService {await workspaceDatabase.updateTable('file_states').set({- upload_status: 'failed',+ upload_status: UploadStatus.Failed,updated_at: new Date().toISOString(),}).where('file_id', '=', upload.file_id)@@ -211,11 +215,44 @@ class FileService {const presignedUrl = data.url;const fileStream = fs.createReadStream(filePath);++ let lastProgress = 0;await axios.put(presignedUrl, fileStream, {headers: {'Content-Type': file.mime_type,'Content-Length': file.size,},+ onUploadProgress: async (progressEvent) => {+ const progress = Math.round(+ (progressEvent.loaded / file.size) * 100+ );++ if (progress >= lastProgress) {+ return;+ }++ lastProgress = progress;++ const updatedFileState = await workspaceDatabase+ .updateTable('file_states')+ .returningAll()+ .set({+ upload_progress: progress,+ updated_at: new Date().toISOString(),+ })+ .where('file_id', '=', file.id)+ .executeTakeFirst();++ if (!updatedFileState) {+ return;+ }++ eventBus.publish({+ type: 'file_state_updated',+ userId,+ fileState: mapFileState(updatedFileState),+ });+ },});const { status } = await httpClient.put<CompleteUploadOutput>(@@ -234,7 +271,7 @@ class FileService {await workspaceDatabase.updateTable('file_states').set({- upload_status: 'completed',+ upload_status: UploadStatus.Completed,upload_progress: 100,updated_at: new Date().toISOString(),})@@ -251,13 +288,15 @@ class FileService {}public async downloadFiles(userId: string): Promise<void> {+ this.debug(`Downloading files for user ${userId}`);+const workspaceDatabase =await databaseService.getWorkspaceDatabase(userId);const downloads = await workspaceDatabase.selectFrom('file_states').selectAll()- .where('download_status', '=', 'pending')+ .where('download_status', '=', DownloadStatus.Pending).execute();if (downloads.length === 0) {@@ -309,7 +348,7 @@ class FileService {.updateTable('file_states').returningAll().set({- download_status: 'completed',+ download_status: DownloadStatus.Completed,download_progress: 100,updated_at: new Date().toISOString(),})@@ -344,8 +383,43 @@ class FileService {const presignedUrl = data.url;const fileStream = fs.createWriteStream(filePath);+ let lastProgress = 0;+await axios- .get(presignedUrl, { responseType: 'stream' })+ .get(presignedUrl, {+ responseType: 'stream',+ onDownloadProgress: async (progressEvent) => {+ const progress = Math.round(+ (progressEvent.loaded / file.size) * 100+ );++ if (progress <= lastProgress) {+ return;+ }++ lastProgress = progress;++ const updatedFileState = await workspaceDatabase+ .updateTable('file_states')+ .returningAll()+ .set({+ download_progress: progress,+ updated_at: new Date().toISOString(),+ })+ .where('file_id', '=', file.id)+ .executeTakeFirst();++ if (!updatedFileState) {+ return;+ }++ eventBus.publish({+ type: 'file_state_updated',+ userId,+ fileState: mapFileState(updatedFileState),+ });+ },+ }).then((response) => {response.data.pipe(fileStream);});@@ -354,7 +428,7 @@ class FileService {.updateTable('file_states').returningAll().set({- download_status: 'completed',+ download_status: DownloadStatus.Completed,download_progress: 100,updated_at: new Date().toISOString(),})diff --git a/apps/desktop/src/renderer/components/files/file-download.tsx b/apps/desktop/src/renderer/components/files/file-download.tsxindex 6a719b34..a9f284dd 100644--- a/apps/desktop/src/renderer/components/files/file-download.tsx+++ b/apps/desktop/src/renderer/components/files/file-download.tsx@@ -4,7 +4,7 @@ import { Spinner } from '@/renderer/components/ui/spinner';import { useWorkspace } from '@/renderer/contexts/workspace';import { useMutation } from '@/renderer/hooks/use-mutation';import { toast } from '@/renderer/hooks/use-toast';-import { FileWithState } from '@/shared/types/files';+import { DownloadStatus, FileWithState } from '@/shared/types/files';import { formatBytes } from '@/shared/lib/files';interface FileDownloadProps {@@ -15,14 +15,14 @@ export const FileDownload = ({ file }: FileDownloadProps) => {const workspace = useWorkspace();const { mutate } = useMutation();- const isDownloading = file.downloadStatus === 'pending';+ const isDownloading = file.downloadStatus === DownloadStatus.Pending;return (<div className="flex h-full w-full items-center justify-center">{isDownloading ? (<div className="flex flex-col items-center gap-3 text-muted-foreground"><Spinner className="size-8" />- <p className="text-sm">Downloading file...</p>+ <p className="text-sm">Downloading file ({file.downloadProgress}%)</p></div>) : (<divdiff --git a/apps/desktop/src/shared/mutations/index.ts b/apps/desktop/src/shared/mutations/index.tsindex 061ccda0..3858ea13 100644--- a/apps/desktop/src/shared/mutations/index.ts+++ b/apps/desktop/src/shared/mutations/index.ts@@ -96,6 +96,7 @@ export enum MutationErrorCode {FileCreateForbidden = 'file_create_forbidden',FileCreateFailed = 'file_create_failed',FileNotFound = 'file_not_found',+ FileNotReady = 'file_not_ready',FileDeleteForbidden = 'file_delete_forbidden',FileDeleteFailed = 'file_delete_failed',FolderUpdateForbidden = 'folder_update_forbidden',diff --git a/apps/desktop/src/shared/types/files.ts b/apps/desktop/src/shared/types/files.tsindex d830a992..a31a0fac 100644--- a/apps/desktop/src/shared/types/files.ts+++ b/apps/desktop/src/shared/types/files.ts@@ -28,8 +28,19 @@ export type File = {version: bigint;};-export type DownloadStatus = 'none' | 'pending' | 'completed' | 'failed';-export type UploadStatus = 'none' | 'pending' | 'completed' | 'failed';+export enum DownloadStatus {+ None = 0,+ Pending = 1,+ Completed = 2,+ Failed = 3,+}++export enum UploadStatus {+ None = 0,+ Pending = 1,+ Completed = 2,+ Failed = 3,+}export type FileState = {fileId: string;diff --git a/apps/server/src/controllers/client/workspaces/files/file-download-get.ts b/apps/server/src/controllers/client/workspaces/files/file-download-get.tsindex 89a046e7..a30f0628 100644--- a/apps/server/src/controllers/client/workspaces/files/file-download-get.ts+++ b/apps/server/src/controllers/client/workspaces/files/file-download-get.ts@@ -4,6 +4,7 @@ import {hasEntryRole,ApiErrorCode,extractEntryRole,+ FileStatus,} from '@colanode/core';import { GetObjectCommand } from '@aws-sdk/client-s3';import { getSignedUrl } from '@aws-sdk/s3-request-presigner';@@ -33,6 +34,13 @@ export const fileDownloadGetHandler = async (});}+ if (file.status !== FileStatus.Ready) {+ return ResponseBuilder.badRequest(res, {+ code: ApiErrorCode.FileNotReady,+ message: 'File is not ready to be downloaded.',+ });+ }+const root = await fetchEntry(file.root_id);if (!root) {return ResponseBuilder.badRequest(res, {diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.tsindex 0103618d..0b918b8e 100644--- a/packages/core/src/types/api.ts+++ b/packages/core/src/types/api.ts@@ -26,6 +26,7 @@ export enum ApiErrorCode {WorkspaceUpdateFailed = 'workspace_update_failed',FileNotFound = 'file_not_found',FileNoAccess = 'file_no_access',+ FileNotReady = 'file_not_ready',FileOwnerMismatch = 'file_owner_mismatch',WorkspaceMismatch = 'workspace_mismatch',FileError = 'file_error',