Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
Implement file jobs and scheduler improvements in desktop
Hakan Shehudfcf13bda187
diff --git a/apps/desktop/src/main/jobs/clean-deleted-files.ts b/apps/desktop/src/main/jobs/clean-deleted-files.tsnew file mode 100644index 00000000..ca39f36b--- /dev/null+++ b/apps/desktop/src/main/jobs/clean-deleted-files.ts@@ -0,0 +1,26 @@+import { fileService } from '@/main/services/file-service';+import { JobHandler } from '@/main/jobs';++export type CleanDeletedFilesInput = {+ type: 'clean_deleted_files';+ userId: string;+};++declare module '@/main/jobs' {+ interface JobMap {+ clean_deleted_files: {+ input: CleanDeletedFilesInput;+ };+ }+}++export class CleanDeletedFilesJobHandler+ implements JobHandler<CleanDeletedFilesInput>+{+ public triggerDebounce = 0;+ public interval = 1000 * 60;++ public async handleJob(input: CleanDeletedFilesInput) {+ await fileService.cleanDeletedFiles(input.userId);+ }+}diff --git a/apps/desktop/src/main/jobs/download-files.ts b/apps/desktop/src/main/jobs/download-files.tsnew file mode 100644index 00000000..fc778303--- /dev/null+++ b/apps/desktop/src/main/jobs/download-files.ts@@ -0,0 +1,24 @@+import { fileService } from '@/main/services/file-service';+import { JobHandler } from '@/main/jobs';++export type DownloadFilesInput = {+ type: 'download_files';+ userId: string;+};++declare module '@/main/jobs' {+ interface JobMap {+ download_files: {+ input: DownloadFilesInput;+ };+ }+}++export class DownloadFilesJobHandler implements JobHandler<DownloadFilesInput> {+ public triggerDebounce = 0;+ public interval = 1000 * 60;++ public async handleJob(input: DownloadFilesInput) {+ await fileService.downloadFiles(input.userId);+ }+}diff --git a/apps/desktop/src/main/jobs/upload-files.ts b/apps/desktop/src/main/jobs/upload-files.tsnew file mode 100644index 00000000..33668312--- /dev/null+++ b/apps/desktop/src/main/jobs/upload-files.ts@@ -0,0 +1,24 @@+import { fileService } from '@/main/services/file-service';+import { JobHandler } from '@/main/jobs';++export type UploadFilesInput = {+ type: 'upload_files';+ userId: string;+};++declare module '@/main/jobs' {+ interface JobMap {+ upload_files: {+ input: UploadFilesInput;+ };+ }+}++export class UploadFilesJobHandler implements JobHandler<UploadFilesInput> {+ public triggerDebounce = 0;+ public interval = 1000 * 60;++ public async handleJob(input: UploadFilesInput) {+ await fileService.uploadFiles(input.userId);+ }+}diff --git a/apps/desktop/src/main/scheduler.ts b/apps/desktop/src/main/scheduler.tsindex de5c169e..4949ca90 100644--- a/apps/desktop/src/main/scheduler.ts+++ b/apps/desktop/src/main/scheduler.ts@@ -15,6 +15,9 @@ import { SyncPendingInteractionsJobHandler } from '@/main/jobs/sync-pending-inteimport { SyncMissingNodesJobHandler } from '@/main/jobs/sync-missing-nodes';import { SyncDeletedTokensJobHandler } from '@/main/jobs/sync-deleted-tokens';import { ConnectSocketJobHandler } from '@/main/jobs/connect-socket';+import { UploadFilesJobHandler } from '@/main/jobs/upload-files';+import { DownloadFilesJobHandler } from '@/main/jobs/download-files';+import { CleanDeletedFilesJobHandler } from '@/main/jobs/clean-deleted-files';import { eventBus } from '@/shared/lib/event-bus';import { Event } from '@/shared/types/events';@@ -33,6 +36,9 @@ export const jobHandlerMap: JobHandlerMap = {sync_missing_nodes: new SyncMissingNodesJobHandler(),sync_deleted_tokens: new SyncDeletedTokensJobHandler(),connect_socket: new ConnectSocketJobHandler(),+ upload_files: new UploadFilesJobHandler(),+ download_files: new DownloadFilesJobHandler(),+ clean_deleted_files: new CleanDeletedFilesJobHandler(),};type JobState = {@@ -147,7 +153,15 @@ class Scheduler {} finally {state.running = false;- if (state.timeout === null && state.handler.interval > 0) {+ if (state.timeout) {+ clearTimeout(state.timeout);+ }++ if (state.triggered) {+ state.timeout = setTimeout(() => {+ this.executeJob(state);+ }, state.handler.triggerDebounce);+ } else if (state.handler.interval > 0) {state.timeout = setTimeout(() => {this.executeJob(state);}, state.handler.interval);@@ -247,6 +261,16 @@ class Scheduler {userId,accountId,});++ this.schedule({+ type: 'upload_files',+ userId,+ });++ this.schedule({+ type: 'download_files',+ userId,+ });}private deleteWorkspaceJobs(userId: string) {@@ -258,48 +282,78 @@ class Scheduler {continue;}- if (- state.input.type === 'init_sync_consumers' &&- state.input.userId === userId- ) {- this.states.delete(jobId);- }+ if (this.isWorkspaceJob(state, userId)) {+ if (state.timeout) {+ clearTimeout(state.timeout);+ }- if (- state.input.type === 'sync_pending_transactions' &&- state.input.userId === userId- ) {this.states.delete(jobId);}+ }+ }- if (- state.input.type === 'sync_incomplete_transactions' &&- state.input.userId === userId- ) {- this.states.delete(jobId);- }+ private isWorkspaceJob(state: JobState, userId: string) {+ if (+ state.input.type === 'init_sync_consumers' &&+ state.input.userId === userId+ ) {+ return true;+ }- if (- state.input.type === 'sync_pending_interactions' &&- state.input.userId === userId- ) {- this.states.delete(jobId);- }+ if (+ state.input.type === 'sync_pending_transactions' &&+ state.input.userId === userId+ ) {+ return true;+ }- if (- state.input.type === 'sync_missing_nodes' &&- state.input.userId === userId- ) {- this.states.delete(jobId);- }+ if (+ state.input.type === 'sync_incomplete_transactions' &&+ state.input.userId === userId+ ) {+ return true;+ }- if (- state.input.type === 'revert_invalid_transactions' &&- state.input.userId === userId- ) {- this.states.delete(jobId);- }+ if (+ state.input.type === 'sync_pending_interactions' &&+ state.input.userId === userId+ ) {+ return true;+ }++ if (+ state.input.type === 'sync_missing_nodes' &&+ state.input.userId === userId+ ) {+ return true;+ }++ if (+ state.input.type === 'revert_invalid_transactions' &&+ state.input.userId === userId+ ) {+ return true;+ }++ if (state.input.type === 'upload_files' && state.input.userId === userId) {+ return true;}++ if (+ state.input.type === 'download_files' &&+ state.input.userId === userId+ ) {+ return true;+ }++ if (+ state.input.type === 'clean_deleted_files' &&+ state.input.userId === userId+ ) {+ return true;+ }++ return false;}private handleEvent(event: Event) {@@ -342,6 +396,39 @@ class Scheduler {type: 'sync_pending_interactions',userId: event.userId,});+ } else if (event.type === 'node_created' && event.node.type === 'file') {+ this.trigger({+ type: 'upload_files',+ userId: event.userId,+ });+ this.trigger({+ type: 'download_files',+ userId: event.userId,+ });+ } else if (event.type === 'node_updated' && event.node.type === 'file') {+ this.trigger({+ type: 'upload_files',+ userId: event.userId,+ });+ this.trigger({+ type: 'download_files',+ userId: event.userId,+ });+ } else if (event.type === 'node_deleted' && event.node.type === 'file') {+ this.trigger({+ type: 'clean_deleted_files',+ userId: event.userId,+ });+ } else if (event.type === 'download_created') {+ this.trigger({+ type: 'download_files',+ userId: event.userId,+ });+ } else if (event.type === 'upload_created') {+ this.trigger({+ type: 'upload_files',+ userId: event.userId,+ });}}}diff --git a/apps/desktop/src/main/services/file-service.ts b/apps/desktop/src/main/services/file-service.tsindex 20c11dc0..cd7be659 100644--- a/apps/desktop/src/main/services/file-service.ts+++ b/apps/desktop/src/main/services/file-service.ts@@ -23,35 +23,8 @@ import { eventBus } from '@/shared/lib/event-bus';import { httpClient } from '@/shared/lib/http-client';import { FileMetadata } from '@/shared/types/files';-type WorkspaceFileState = {- isUploading: boolean;- isDownloading: boolean;- isUploadScheduled: boolean;- isDownloadScheduled: boolean;-};-class FileService {private readonly debug = createDebugger('service:file');- private readonly fileStates: Map<string, WorkspaceFileState> = new Map();-- constructor() {- eventBus.subscribe((event) => {- if (event.type === 'download_created') {- this.syncWorkspaceDownloads(event.userId);- } else if (event.type === 'upload_created') {- this.syncWorkspaceUploads(event.userId);- } else if (event.type === 'node_created' && event.node.type === 'file') {- this.syncWorkspaceDownloads(event.userId);- this.syncWorkspaceUploads(event.userId);- } else if (event.type === 'node_updated' && event.node.type === 'file') {- this.syncWorkspaceDownloads(event.userId);- this.syncWorkspaceUploads(event.userId);- } else if (event.type === 'node_deleted' && event.node.type === 'file') {- this.syncWorkspaceDownloads(event.userId);- this.syncWorkspaceUploads(event.userId);- }- });- }public async handleFileRequest(request: Request): Promise<Response> {const url = request.url.replace('local-file://', '');@@ -143,88 +116,7 @@ class FileService {};}- public async syncFiles() {- this.debug('Syncing files');-- const workspaces = await databaseService.appDatabase- .selectFrom('workspaces')- .select(['user_id'])- .execute();-- for (const workspace of workspaces) {- await this.syncWorkspaceUploads(workspace.user_id);- await this.syncWorkspaceDownloads(workspace.user_id);- await this.checkDeletedFiles(workspace.user_id);- }- }-- public async syncWorkspaceUploads(userId: string): Promise<void> {- if (!this.fileStates.has(userId)) {- this.fileStates.set(userId, {- isUploading: false,- isDownloading: false,- isUploadScheduled: false,- isDownloadScheduled: false,- });- }-- const fileState = this.fileStates.get(userId)!;- if (fileState.isUploading) {- fileState.isUploadScheduled = true;- this.debug(- `Uploading files for user ${userId} is in progress, scheduling upload`- );- return;- }-- fileState.isUploading = true;- try {- await this.uploadWorkspaceFiles(userId);- } catch (error) {- this.debug(error, `Error uploading files for user ${userId}`);- } finally {- fileState.isUploading = false;- if (fileState.isUploadScheduled) {- fileState.isUploadScheduled = false;- this.syncWorkspaceUploads(userId);- }- }- }-- public async syncWorkspaceDownloads(userId: string): Promise<void> {- if (!this.fileStates.has(userId)) {- this.fileStates.set(userId, {- isUploading: false,- isDownloading: false,- isUploadScheduled: false,- isDownloadScheduled: false,- });- }-- const fileState = this.fileStates.get(userId)!;- if (fileState.isDownloading) {- fileState.isDownloadScheduled = true;- this.debug(- `Downloading files for user ${userId} is in progress, scheduling download`- );- return;- }-- fileState.isDownloading = true;- try {- await this.downloadWorkspaceFiles(userId);- } catch (error) {- this.debug(error, `Error downloading files for user ${userId}`);- } finally {- fileState.isDownloading = false;- if (fileState.isDownloadScheduled) {- fileState.isDownloadScheduled = false;- this.syncWorkspaceDownloads(userId);- }- }- }-- private async uploadWorkspaceFiles(userId: string): Promise<void> {+ public async uploadFiles(userId: string): Promise<void> {this.debug(`Uploading files for user ${userId}`);const workspaceDatabase =@@ -355,7 +247,7 @@ class FileService {}}- public async downloadWorkspaceFiles(userId: string): Promise<void> {+ public async downloadFiles(userId: string): Promise<void> {const workspaceDatabase =await databaseService.getWorkspaceDatabase(userId);@@ -505,7 +397,7 @@ class FileService {}}- private async checkDeletedFiles(userId: string): Promise<void> {+ public async cleanDeletedFiles(userId: string): Promise<void> {this.debug(`Checking deleted files for user ${userId}`);const workspaceDatabase =