Skip to content

Open-source and local-first Slack and Notion alternative that puts you in control of your data

Implement file jobs and scheduler improvements in desktop

Hakan Shehudfcf13bda187
diff --git a/apps/desktop/src/main/jobs/clean-deleted-files.ts b/apps/desktop/src/main/jobs/clean-deleted-files.ts
new file mode 100644
index 00000000..ca39f36b
--- /dev/null
+++ b/apps/desktop/src/main/jobs/clean-deleted-files.ts
@@ -0,0 +1,26 @@
+import { fileService } from '@/main/services/file-service';
+import { JobHandler } from '@/main/jobs';
+
+export type CleanDeletedFilesInput = {
+ type: 'clean_deleted_files';
+ userId: string;
+};
+
+declare module '@/main/jobs' {
+ interface JobMap {
+ clean_deleted_files: {
+ input: CleanDeletedFilesInput;
+ };
+ }
+}
+
+export class CleanDeletedFilesJobHandler
+ implements JobHandler<CleanDeletedFilesInput>
+{
+ public triggerDebounce = 0;
+ public interval = 1000 * 60;
+
+ public async handleJob(input: CleanDeletedFilesInput) {
+ await fileService.cleanDeletedFiles(input.userId);
+ }
+}
diff --git a/apps/desktop/src/main/jobs/download-files.ts b/apps/desktop/src/main/jobs/download-files.ts
new file mode 100644
index 00000000..fc778303
--- /dev/null
+++ b/apps/desktop/src/main/jobs/download-files.ts
@@ -0,0 +1,24 @@
+import { fileService } from '@/main/services/file-service';
+import { JobHandler } from '@/main/jobs';
+
+export type DownloadFilesInput = {
+ type: 'download_files';
+ userId: string;
+};
+
+declare module '@/main/jobs' {
+ interface JobMap {
+ download_files: {
+ input: DownloadFilesInput;
+ };
+ }
+}
+
+export class DownloadFilesJobHandler implements JobHandler<DownloadFilesInput> {
+ public triggerDebounce = 0;
+ public interval = 1000 * 60;
+
+ public async handleJob(input: DownloadFilesInput) {
+ await fileService.downloadFiles(input.userId);
+ }
+}
diff --git a/apps/desktop/src/main/jobs/upload-files.ts b/apps/desktop/src/main/jobs/upload-files.ts
new file mode 100644
index 00000000..33668312
--- /dev/null
+++ b/apps/desktop/src/main/jobs/upload-files.ts
@@ -0,0 +1,24 @@
+import { fileService } from '@/main/services/file-service';
+import { JobHandler } from '@/main/jobs';
+
+export type UploadFilesInput = {
+ type: 'upload_files';
+ userId: string;
+};
+
+declare module '@/main/jobs' {
+ interface JobMap {
+ upload_files: {
+ input: UploadFilesInput;
+ };
+ }
+}
+
+export class UploadFilesJobHandler implements JobHandler<UploadFilesInput> {
+ public triggerDebounce = 0;
+ public interval = 1000 * 60;
+
+ public async handleJob(input: UploadFilesInput) {
+ await fileService.uploadFiles(input.userId);
+ }
+}
diff --git a/apps/desktop/src/main/scheduler.ts b/apps/desktop/src/main/scheduler.ts
index de5c169e..4949ca90 100644
--- a/apps/desktop/src/main/scheduler.ts
+++ b/apps/desktop/src/main/scheduler.ts
@@ -15,6 +15,9 @@ import { SyncPendingInteractionsJobHandler } from '@/main/jobs/sync-pending-inte
import { SyncMissingNodesJobHandler } from '@/main/jobs/sync-missing-nodes';
import { SyncDeletedTokensJobHandler } from '@/main/jobs/sync-deleted-tokens';
import { ConnectSocketJobHandler } from '@/main/jobs/connect-socket';
+import { UploadFilesJobHandler } from '@/main/jobs/upload-files';
+import { DownloadFilesJobHandler } from '@/main/jobs/download-files';
+import { CleanDeletedFilesJobHandler } from '@/main/jobs/clean-deleted-files';
import { eventBus } from '@/shared/lib/event-bus';
import { Event } from '@/shared/types/events';
@@ -33,6 +36,9 @@ export const jobHandlerMap: JobHandlerMap = {
sync_missing_nodes: new SyncMissingNodesJobHandler(),
sync_deleted_tokens: new SyncDeletedTokensJobHandler(),
connect_socket: new ConnectSocketJobHandler(),
+ upload_files: new UploadFilesJobHandler(),
+ download_files: new DownloadFilesJobHandler(),
+ clean_deleted_files: new CleanDeletedFilesJobHandler(),
};
type JobState = {
@@ -147,7 +153,15 @@ class Scheduler {
} finally {
state.running = false;
- if (state.timeout === null && state.handler.interval > 0) {
+ if (state.timeout) {
+ clearTimeout(state.timeout);
+ }
+
+ if (state.triggered) {
+ state.timeout = setTimeout(() => {
+ this.executeJob(state);
+ }, state.handler.triggerDebounce);
+ } else if (state.handler.interval > 0) {
state.timeout = setTimeout(() => {
this.executeJob(state);
}, state.handler.interval);
@@ -247,6 +261,16 @@ class Scheduler {
userId,
accountId,
});
+
+ this.schedule({
+ type: 'upload_files',
+ userId,
+ });
+
+ this.schedule({
+ type: 'download_files',
+ userId,
+ });
}
private deleteWorkspaceJobs(userId: string) {
@@ -258,48 +282,78 @@ class Scheduler {
continue;
}
- if (
- state.input.type === 'init_sync_consumers' &&
- state.input.userId === userId
- ) {
- this.states.delete(jobId);
- }
+ if (this.isWorkspaceJob(state, userId)) {
+ if (state.timeout) {
+ clearTimeout(state.timeout);
+ }
- if (
- state.input.type === 'sync_pending_transactions' &&
- state.input.userId === userId
- ) {
this.states.delete(jobId);
}
+ }
+ }
- if (
- state.input.type === 'sync_incomplete_transactions' &&
- state.input.userId === userId
- ) {
- this.states.delete(jobId);
- }
+ private isWorkspaceJob(state: JobState, userId: string) {
+ if (
+ state.input.type === 'init_sync_consumers' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
- if (
- state.input.type === 'sync_pending_interactions' &&
- state.input.userId === userId
- ) {
- this.states.delete(jobId);
- }
+ if (
+ state.input.type === 'sync_pending_transactions' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
- if (
- state.input.type === 'sync_missing_nodes' &&
- state.input.userId === userId
- ) {
- this.states.delete(jobId);
- }
+ if (
+ state.input.type === 'sync_incomplete_transactions' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
- if (
- state.input.type === 'revert_invalid_transactions' &&
- state.input.userId === userId
- ) {
- this.states.delete(jobId);
- }
+ if (
+ state.input.type === 'sync_pending_interactions' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
+
+ if (
+ state.input.type === 'sync_missing_nodes' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
+
+ if (
+ state.input.type === 'revert_invalid_transactions' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
+
+ if (state.input.type === 'upload_files' && state.input.userId === userId) {
+ return true;
}
+
+ if (
+ state.input.type === 'download_files' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
+
+ if (
+ state.input.type === 'clean_deleted_files' &&
+ state.input.userId === userId
+ ) {
+ return true;
+ }
+
+ return false;
}
private handleEvent(event: Event) {
@@ -342,6 +396,39 @@ class Scheduler {
type: 'sync_pending_interactions',
userId: event.userId,
});
+ } else if (event.type === 'node_created' && event.node.type === 'file') {
+ this.trigger({
+ type: 'upload_files',
+ userId: event.userId,
+ });
+ this.trigger({
+ type: 'download_files',
+ userId: event.userId,
+ });
+ } else if (event.type === 'node_updated' && event.node.type === 'file') {
+ this.trigger({
+ type: 'upload_files',
+ userId: event.userId,
+ });
+ this.trigger({
+ type: 'download_files',
+ userId: event.userId,
+ });
+ } else if (event.type === 'node_deleted' && event.node.type === 'file') {
+ this.trigger({
+ type: 'clean_deleted_files',
+ userId: event.userId,
+ });
+ } else if (event.type === 'download_created') {
+ this.trigger({
+ type: 'download_files',
+ userId: event.userId,
+ });
+ } else if (event.type === 'upload_created') {
+ this.trigger({
+ type: 'upload_files',
+ userId: event.userId,
+ });
}
}
}
diff --git a/apps/desktop/src/main/services/file-service.ts b/apps/desktop/src/main/services/file-service.ts
index 20c11dc0..cd7be659 100644
--- a/apps/desktop/src/main/services/file-service.ts
+++ b/apps/desktop/src/main/services/file-service.ts
@@ -23,35 +23,8 @@ import { eventBus } from '@/shared/lib/event-bus';
import { httpClient } from '@/shared/lib/http-client';
import { FileMetadata } from '@/shared/types/files';
-type WorkspaceFileState = {
- isUploading: boolean;
- isDownloading: boolean;
- isUploadScheduled: boolean;
- isDownloadScheduled: boolean;
-};
-
class FileService {
private readonly debug = createDebugger('service:file');
- private readonly fileStates: Map<string, WorkspaceFileState> = new Map();
-
- constructor() {
- eventBus.subscribe((event) => {
- if (event.type === 'download_created') {
- this.syncWorkspaceDownloads(event.userId);
- } else if (event.type === 'upload_created') {
- this.syncWorkspaceUploads(event.userId);
- } else if (event.type === 'node_created' && event.node.type === 'file') {
- this.syncWorkspaceDownloads(event.userId);
- this.syncWorkspaceUploads(event.userId);
- } else if (event.type === 'node_updated' && event.node.type === 'file') {
- this.syncWorkspaceDownloads(event.userId);
- this.syncWorkspaceUploads(event.userId);
- } else if (event.type === 'node_deleted' && event.node.type === 'file') {
- this.syncWorkspaceDownloads(event.userId);
- this.syncWorkspaceUploads(event.userId);
- }
- });
- }
public async handleFileRequest(request: Request): Promise<Response> {
const url = request.url.replace('local-file://', '');
@@ -143,88 +116,7 @@ class FileService {
};
}
- public async syncFiles() {
- this.debug('Syncing files');
-
- const workspaces = await databaseService.appDatabase
- .selectFrom('workspaces')
- .select(['user_id'])
- .execute();
-
- for (const workspace of workspaces) {
- await this.syncWorkspaceUploads(workspace.user_id);
- await this.syncWorkspaceDownloads(workspace.user_id);
- await this.checkDeletedFiles(workspace.user_id);
- }
- }
-
- public async syncWorkspaceUploads(userId: string): Promise<void> {
- if (!this.fileStates.has(userId)) {
- this.fileStates.set(userId, {
- isUploading: false,
- isDownloading: false,
- isUploadScheduled: false,
- isDownloadScheduled: false,
- });
- }
-
- const fileState = this.fileStates.get(userId)!;
- if (fileState.isUploading) {
- fileState.isUploadScheduled = true;
- this.debug(
- `Uploading files for user ${userId} is in progress, scheduling upload`
- );
- return;
- }
-
- fileState.isUploading = true;
- try {
- await this.uploadWorkspaceFiles(userId);
- } catch (error) {
- this.debug(error, `Error uploading files for user ${userId}`);
- } finally {
- fileState.isUploading = false;
- if (fileState.isUploadScheduled) {
- fileState.isUploadScheduled = false;
- this.syncWorkspaceUploads(userId);
- }
- }
- }
-
- public async syncWorkspaceDownloads(userId: string): Promise<void> {
- if (!this.fileStates.has(userId)) {
- this.fileStates.set(userId, {
- isUploading: false,
- isDownloading: false,
- isUploadScheduled: false,
- isDownloadScheduled: false,
- });
- }
-
- const fileState = this.fileStates.get(userId)!;
- if (fileState.isDownloading) {
- fileState.isDownloadScheduled = true;
- this.debug(
- `Downloading files for user ${userId} is in progress, scheduling download`
- );
- return;
- }
-
- fileState.isDownloading = true;
- try {
- await this.downloadWorkspaceFiles(userId);
- } catch (error) {
- this.debug(error, `Error downloading files for user ${userId}`);
- } finally {
- fileState.isDownloading = false;
- if (fileState.isDownloadScheduled) {
- fileState.isDownloadScheduled = false;
- this.syncWorkspaceDownloads(userId);
- }
- }
- }
-
- private async uploadWorkspaceFiles(userId: string): Promise<void> {
+ public async uploadFiles(userId: string): Promise<void> {
this.debug(`Uploading files for user ${userId}`);
const workspaceDatabase =
@@ -355,7 +247,7 @@ class FileService {
}
}
- public async downloadWorkspaceFiles(userId: string): Promise<void> {
+ public async downloadFiles(userId: string): Promise<void> {
const workspaceDatabase =
await databaseService.getWorkspaceDatabase(userId);
@@ -505,7 +397,7 @@ class FileService {
}
}
- private async checkDeletedFiles(userId: string): Promise<void> {
+ public async cleanDeletedFiles(userId: string): Promise<void> {
this.debug(`Checking deleted files for user ${userId}`);
const workspaceDatabase =