Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Improve file state handling

Hakan Shehu4fb71bcaad40
diff --git a/apps/desktop/src/main/data/workspace/migrations.ts b/apps/desktop/src/main/data/workspace/migrations.ts
index c576fde1..1b131b32 100644
--- a/apps/desktop/src/main/data/workspace/migrations.ts
+++ b/apps/desktop/src/main/data/workspace/migrations.ts
@@ -238,10 +238,10 @@ const createFileStatesTable: Migration = {
await db.schema
.createTable('file_states')
.addColumn('file_id', 'text', (col) => col.notNull().primaryKey())
- .addColumn('download_status', 'text', (col) => col.notNull())
+ .addColumn('download_status', 'integer', (col) => col.notNull())
.addColumn('download_progress', 'integer', (col) => col.notNull())
.addColumn('download_retries', 'integer', (col) => col.notNull())
- .addColumn('upload_status', 'text', (col) => col.notNull())
+ .addColumn('upload_status', 'integer', (col) => col.notNull())
.addColumn('upload_progress', 'integer', (col) => col.notNull())
.addColumn('upload_retries', 'integer', (col) => col.notNull())
.addColumn('created_at', 'text', (col) => col.notNull())
diff --git a/apps/desktop/src/main/mutations/files/file-create.ts b/apps/desktop/src/main/mutations/files/file-create.ts
index 45d108cd..bd63dcf6 100644
--- a/apps/desktop/src/main/mutations/files/file-create.ts
+++ b/apps/desktop/src/main/mutations/files/file-create.ts
@@ -23,6 +23,7 @@ import {
mapFile,
} from '@/main/utils';
import { formatBytes } from '@/shared/lib/files';
+import { DownloadStatus, UploadStatus } from '@/shared/types/files';
export class FileCreateMutationHandler
implements MutationHandler<FileCreateMutationInput>
@@ -167,10 +168,10 @@ export class FileCreateMutationHandler
file_id: fileId,
created_at: new Date().toISOString(),
download_progress: 100,
- download_status: 'completed',
+ download_status: DownloadStatus.Completed,
download_retries: 0,
upload_progress: 0,
- upload_status: 'pending',
+ upload_status: UploadStatus.Pending,
upload_retries: 0,
})
.execute();
@@ -211,10 +212,10 @@ export class FileCreateMutationHandler
fileState: {
fileId: fileId,
downloadProgress: 100,
- downloadStatus: 'completed',
+ downloadStatus: DownloadStatus.Completed,
downloadRetries: 0,
uploadProgress: 10,
- uploadStatus: 'pending',
+ uploadStatus: UploadStatus.Pending,
uploadRetries: 0,
createdAt: new Date().toISOString(),
updatedAt: null,
diff --git a/apps/desktop/src/main/mutations/files/file-download.ts b/apps/desktop/src/main/mutations/files/file-download.ts
index dba41abf..5f486c26 100644
--- a/apps/desktop/src/main/mutations/files/file-download.ts
+++ b/apps/desktop/src/main/mutations/files/file-download.ts
@@ -1,3 +1,5 @@
+import { FileStatus } from '@colanode/core';
+
import { databaseService } from '@/main/data/database-service';
import { MutationHandler } from '@/main/types';
import { mapFileState } from '@/main/utils';
@@ -7,6 +9,7 @@ import {
FileDownloadMutationInput,
FileDownloadMutationOutput,
} from '@/shared/mutations/files/file-download';
+import { DownloadStatus, UploadStatus } from '@/shared/types/files';
export class FileDownloadMutationHandler
implements MutationHandler<FileDownloadMutationInput>
@@ -31,6 +34,13 @@ export class FileDownloadMutationHandler
);
}
+ if (file.status !== FileStatus.Ready) {
+ throw new MutationError(
+ MutationErrorCode.FileNotReady,
+ 'The file you are trying to download is not uploaded by the author yet.'
+ );
+ }
+
const existingFileState = await workspaceDatabase
.selectFrom('file_states')
.selectAll()
@@ -39,7 +49,7 @@ export class FileDownloadMutationHandler
if (
existingFileState &&
- existingFileState.download_status === 'completed'
+ existingFileState.download_status === DownloadStatus.Completed
) {
return {
success: true,
@@ -51,17 +61,17 @@ export class FileDownloadMutationHandler
.returningAll()
.values({
file_id: input.fileId,
- download_status: 'pending',
+ download_status: DownloadStatus.Pending,
download_progress: 0,
download_retries: 0,
- upload_status: 'none',
+ upload_status: UploadStatus.None,
upload_progress: 0,
upload_retries: 0,
created_at: new Date().toISOString(),
})
.onConflict((oc) =>
oc.doUpdateSet(() => ({
- download_status: 'pending',
+ download_status: DownloadStatus.Pending,
download_progress: 0,
download_retries: 0,
updated_at: new Date().toISOString(),
diff --git a/apps/desktop/src/main/mutations/messages/message-create.ts b/apps/desktop/src/main/mutations/messages/message-create.ts
index ce957908..73f18f1a 100644
--- a/apps/desktop/src/main/mutations/messages/message-create.ts
+++ b/apps/desktop/src/main/mutations/messages/message-create.ts
@@ -36,6 +36,7 @@ import {
mapMessage,
} from '@/main/utils';
import { formatBytes } from '@/shared/lib/files';
+import { DownloadStatus, UploadStatus } from '@/shared/types/files';
export class MessageCreateMutationHandler
implements MutationHandler<MessageCreateMutationInput>
@@ -170,11 +171,11 @@ export class MessageCreateMutationHandler
fileStates.push({
file_id: fileId,
- upload_status: 'pending',
+ upload_status: UploadStatus.Pending,
upload_progress: 0,
upload_retries: 0,
- download_status: 'pending',
- download_progress: 0,
+ download_status: DownloadStatus.Completed,
+ download_progress: 100,
download_retries: 0,
created_at: createdAt,
});
diff --git a/apps/desktop/src/main/queries/files/file-get.ts b/apps/desktop/src/main/queries/files/file-get.ts
index b11baad3..9bba92bb 100644
--- a/apps/desktop/src/main/queries/files/file-get.ts
+++ b/apps/desktop/src/main/queries/files/file-get.ts
@@ -3,7 +3,11 @@ import { ChangeCheckResult, QueryHandler } from '@/main/types';
import { mapFile } from '@/main/utils';
import { FileGetQueryInput } from '@/shared/queries/files/file-get';
import { Event } from '@/shared/types/events';
-import { FileWithState } from '@/shared/types/files';
+import {
+ DownloadStatus,
+ FileWithState,
+ UploadStatus,
+} from '@/shared/types/files';
export class FileGetQueryHandler implements QueryHandler<FileGetQueryInput> {
public async handleQuery(
@@ -49,9 +53,9 @@ export class FileGetQueryHandler implements QueryHandler<FileGetQueryInput> {
result: {
...event.file,
downloadProgress: output?.downloadProgress ?? 0,
- downloadStatus: output?.downloadStatus ?? 'none',
+ downloadStatus: output?.downloadStatus ?? DownloadStatus.None,
uploadProgress: output?.uploadProgress ?? 0,
- uploadStatus: output?.uploadStatus ?? 'none',
+ uploadStatus: output?.uploadStatus ?? UploadStatus.None,
},
};
}
@@ -160,9 +164,9 @@ export class FileGetQueryHandler implements QueryHandler<FileGetQueryInput> {
const fileWithState: FileWithState = {
...mapFile(file),
downloadProgress: file.download_progress ?? 0,
- downloadStatus: file.download_status ?? 'none',
+ downloadStatus: file.download_status ?? DownloadStatus.None,
uploadProgress: file.upload_progress ?? 0,
- uploadStatus: file.upload_status ?? 'none',
+ uploadStatus: file.upload_status ?? UploadStatus.None,
};
return fileWithState;
diff --git a/apps/desktop/src/main/queries/files/file-list.ts b/apps/desktop/src/main/queries/files/file-list.ts
index 7d552283..d4b54491 100644
--- a/apps/desktop/src/main/queries/files/file-list.ts
+++ b/apps/desktop/src/main/queries/files/file-list.ts
@@ -3,7 +3,11 @@ import { ChangeCheckResult, QueryHandler } from '@/main/types';
import { mapFile } from '@/main/utils';
import { FileListQueryInput } from '@/shared/queries/files/file-list';
import { Event } from '@/shared/types/events';
-import { FileWithState } from '@/shared/types/files';
+import {
+ DownloadStatus,
+ FileWithState,
+ UploadStatus,
+} from '@/shared/types/files';
export class FileListQueryHandler implements QueryHandler<FileListQueryInput> {
public async handleQuery(
@@ -171,9 +175,9 @@ export class FileListQueryHandler implements QueryHandler<FileListQueryInput> {
const filesWithState: FileWithState[] = files.map((file) => ({
...mapFile(file),
downloadProgress: file.download_progress ?? 0,
- downloadStatus: file.download_status ?? 'none',
+ downloadStatus: file.download_status ?? DownloadStatus.None,
uploadProgress: file.upload_progress ?? 0,
- uploadStatus: file.upload_status ?? 'none',
+ uploadStatus: file.upload_status ?? UploadStatus.None,
}));
return filesWithState;
diff --git a/apps/desktop/src/main/services/file-service.ts b/apps/desktop/src/main/services/file-service.ts
index cbc8550f..25332d28 100644
--- a/apps/desktop/src/main/services/file-service.ts
+++ b/apps/desktop/src/main/services/file-service.ts
@@ -27,7 +27,11 @@ import {
} from '@/main/utils';
import { eventBus } from '@/shared/lib/event-bus';
import { httpClient } from '@/shared/lib/http-client';
-import { FileMetadata } from '@/shared/types/files';
+import {
+ DownloadStatus,
+ FileMetadata,
+ UploadStatus,
+} from '@/shared/types/files';
class FileService {
private readonly debug = createDebugger('service:file');
@@ -139,7 +143,7 @@ class FileService {
const uploads = await workspaceDatabase
.selectFrom('file_states')
.selectAll()
- .where('upload_status', '=', 'pending')
+ .where('upload_status', '=', UploadStatus.Pending)
.execute();
if (uploads.length === 0) {
@@ -158,7 +162,7 @@ class FileService {
await workspaceDatabase
.updateTable('file_states')
.set({
- upload_status: 'failed',
+ upload_status: UploadStatus.Failed,
updated_at: new Date().toISOString(),
})
.where('file_id', '=', upload.file_id)
@@ -211,11 +215,44 @@ class FileService {
const presignedUrl = data.url;
const fileStream = fs.createReadStream(filePath);
+
+ let lastProgress = 0;
await axios.put(presignedUrl, fileStream, {
headers: {
'Content-Type': file.mime_type,
'Content-Length': file.size,
},
+ onUploadProgress: async (progressEvent) => {
+ const progress = Math.round(
+ (progressEvent.loaded / file.size) * 100
+ );
+
+ if (progress >= lastProgress) {
+ return;
+ }
+
+ lastProgress = progress;
+
+ const updatedFileState = await workspaceDatabase
+ .updateTable('file_states')
+ .returningAll()
+ .set({
+ upload_progress: progress,
+ updated_at: new Date().toISOString(),
+ })
+ .where('file_id', '=', file.id)
+ .executeTakeFirst();
+
+ if (!updatedFileState) {
+ return;
+ }
+
+ eventBus.publish({
+ type: 'file_state_updated',
+ userId,
+ fileState: mapFileState(updatedFileState),
+ });
+ },
});
const { status } = await httpClient.put<CompleteUploadOutput>(
@@ -234,7 +271,7 @@ class FileService {
await workspaceDatabase
.updateTable('file_states')
.set({
- upload_status: 'completed',
+ upload_status: UploadStatus.Completed,
upload_progress: 100,
updated_at: new Date().toISOString(),
})
@@ -251,13 +288,15 @@ class FileService {
}
public async downloadFiles(userId: string): Promise<void> {
+ this.debug(`Downloading files for user ${userId}`);
+
const workspaceDatabase =
await databaseService.getWorkspaceDatabase(userId);
const downloads = await workspaceDatabase
.selectFrom('file_states')
.selectAll()
- .where('download_status', '=', 'pending')
+ .where('download_status', '=', DownloadStatus.Pending)
.execute();
if (downloads.length === 0) {
@@ -309,7 +348,7 @@ class FileService {
.updateTable('file_states')
.returningAll()
.set({
- download_status: 'completed',
+ download_status: DownloadStatus.Completed,
download_progress: 100,
updated_at: new Date().toISOString(),
})
@@ -344,8 +383,43 @@ class FileService {
const presignedUrl = data.url;
const fileStream = fs.createWriteStream(filePath);
+ let lastProgress = 0;
+
await axios
- .get(presignedUrl, { responseType: 'stream' })
+ .get(presignedUrl, {
+ responseType: 'stream',
+ onDownloadProgress: async (progressEvent) => {
+ const progress = Math.round(
+ (progressEvent.loaded / file.size) * 100
+ );
+
+ if (progress <= lastProgress) {
+ return;
+ }
+
+ lastProgress = progress;
+
+ const updatedFileState = await workspaceDatabase
+ .updateTable('file_states')
+ .returningAll()
+ .set({
+ download_progress: progress,
+ updated_at: new Date().toISOString(),
+ })
+ .where('file_id', '=', file.id)
+ .executeTakeFirst();
+
+ if (!updatedFileState) {
+ return;
+ }
+
+ eventBus.publish({
+ type: 'file_state_updated',
+ userId,
+ fileState: mapFileState(updatedFileState),
+ });
+ },
+ })
.then((response) => {
response.data.pipe(fileStream);
});
@@ -354,7 +428,7 @@ class FileService {
.updateTable('file_states')
.returningAll()
.set({
- download_status: 'completed',
+ download_status: DownloadStatus.Completed,
download_progress: 100,
updated_at: new Date().toISOString(),
})
diff --git a/apps/desktop/src/renderer/components/files/file-download.tsx b/apps/desktop/src/renderer/components/files/file-download.tsx
index 6a719b34..a9f284dd 100644
--- a/apps/desktop/src/renderer/components/files/file-download.tsx
+++ b/apps/desktop/src/renderer/components/files/file-download.tsx
@@ -4,7 +4,7 @@ import { Spinner } from '@/renderer/components/ui/spinner';
import { useWorkspace } from '@/renderer/contexts/workspace';
import { useMutation } from '@/renderer/hooks/use-mutation';
import { toast } from '@/renderer/hooks/use-toast';
-import { FileWithState } from '@/shared/types/files';
+import { DownloadStatus, FileWithState } from '@/shared/types/files';
import { formatBytes } from '@/shared/lib/files';
interface FileDownloadProps {
@@ -15,14 +15,14 @@ export const FileDownload = ({ file }: FileDownloadProps) => {
const workspace = useWorkspace();
const { mutate } = useMutation();
- const isDownloading = file.downloadStatus === 'pending';
+ const isDownloading = file.downloadStatus === DownloadStatus.Pending;
return (
<div className="flex h-full w-full items-center justify-center">
{isDownloading ? (
<div className="flex flex-col items-center gap-3 text-muted-foreground">
<Spinner className="size-8" />
- <p className="text-sm">Downloading file...</p>
+ <p className="text-sm">Downloading file ({file.downloadProgress}%)</p>
</div>
) : (
<div
diff --git a/apps/desktop/src/shared/mutations/index.ts b/apps/desktop/src/shared/mutations/index.ts
index 061ccda0..3858ea13 100644
--- a/apps/desktop/src/shared/mutations/index.ts
+++ b/apps/desktop/src/shared/mutations/index.ts
@@ -96,6 +96,7 @@ export enum MutationErrorCode {
FileCreateForbidden = 'file_create_forbidden',
FileCreateFailed = 'file_create_failed',
FileNotFound = 'file_not_found',
+ FileNotReady = 'file_not_ready',
FileDeleteForbidden = 'file_delete_forbidden',
FileDeleteFailed = 'file_delete_failed',
FolderUpdateForbidden = 'folder_update_forbidden',
diff --git a/apps/desktop/src/shared/types/files.ts b/apps/desktop/src/shared/types/files.ts
index d830a992..a31a0fac 100644
--- a/apps/desktop/src/shared/types/files.ts
+++ b/apps/desktop/src/shared/types/files.ts
@@ -28,8 +28,19 @@ export type File = {
version: bigint;
};
-export type DownloadStatus = 'none' | 'pending' | 'completed' | 'failed';
-export type UploadStatus = 'none' | 'pending' | 'completed' | 'failed';
+export enum DownloadStatus {
+ None = 0,
+ Pending = 1,
+ Completed = 2,
+ Failed = 3,
+}
+
+export enum UploadStatus {
+ None = 0,
+ Pending = 1,
+ Completed = 2,
+ Failed = 3,
+}
export type FileState = {
fileId: string;
diff --git a/apps/server/src/controllers/client/workspaces/files/file-download-get.ts b/apps/server/src/controllers/client/workspaces/files/file-download-get.ts
index 89a046e7..a30f0628 100644
--- a/apps/server/src/controllers/client/workspaces/files/file-download-get.ts
+++ b/apps/server/src/controllers/client/workspaces/files/file-download-get.ts
@@ -4,6 +4,7 @@ import {
hasEntryRole,
ApiErrorCode,
extractEntryRole,
+ FileStatus,
} from '@colanode/core';
import { GetObjectCommand } from '@aws-sdk/client-s3';
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
@@ -33,6 +34,13 @@ export const fileDownloadGetHandler = async (
});
}
+ if (file.status !== FileStatus.Ready) {
+ return ResponseBuilder.badRequest(res, {
+ code: ApiErrorCode.FileNotReady,
+ message: 'File is not ready to be downloaded.',
+ });
+ }
+
const root = await fetchEntry(file.root_id);
if (!root) {
return ResponseBuilder.badRequest(res, {
diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts
index 0103618d..0b918b8e 100644
--- a/packages/core/src/types/api.ts
+++ b/packages/core/src/types/api.ts
@@ -26,6 +26,7 @@ export enum ApiErrorCode {
WorkspaceUpdateFailed = 'workspace_update_failed',
FileNotFound = 'file_not_found',
FileNoAccess = 'file_no_access',
+ FileNotReady = 'file_not_ready',
FileOwnerMismatch = 'file_owner_mismatch',
WorkspaceMismatch = 'workspace_mismatch',
FileError = 'file_error',