Open-source and local-first Slack and Notion alternative that puts you in control of your data
TypeScript chatchat-applicationcrdteditorelectronknowledge-baselocal-firstnotionnotion-alternativerealtime-collaborationself-hostedslacksqliteteam-collaborationwikiyjs
import AsyncLock from 'async-lock';
import { SelectJob, SelectJobSchedule } from '@colanode/client/databases/app';
import {
Job,
JobHandler,
JobHandlerMap,
JobInput,
JobOptions,
JobScheduleOptions,
JobScheduleStatus,
JobStatus,
} from '@colanode/client/jobs';
import { AccountSyncJobHandler } from '@colanode/client/jobs/account-sync';
import { AvatarDownloadJobHandler } from '@colanode/client/jobs/avatar-download';
import { AvatarsCleanJobHandler } from '@colanode/client/jobs/avatars-clean';
import { FileDownloadJobHandler } from '@colanode/client/jobs/file-download';
import { FileUploadJobHandler } from '@colanode/client/jobs/file-upload';
import { LocalFileDownloadJobHandler } from '@colanode/client/jobs/local-file-download';
import { MutationsSyncJobHandler } from '@colanode/client/jobs/mutations-sync';
import { ServerSyncJobHandler } from '@colanode/client/jobs/server-sync';
import { TempFilesCleanJobHandler } from '@colanode/client/jobs/temp-files-clean';
import { TokenDeleteJobHandler } from '@colanode/client/jobs/token-delete';
import { WorkspaceFilesCleanJobHandler } from '@colanode/client/jobs/workspace-files-clean';
import { SleepScheduler } from '@colanode/client/lib/sleep-scheduler';
import { AppService } from '@colanode/client/services/app-service';
import { generateId, IdType } from '@colanode/core';
const MAX_CONCURRENCY = 5;
const JOBS_MAX_TIMEOUT = 30000;
const SCHEDULES_MAX_TIMEOUT = 30000;
const JOBS_CONCURRENCY_TIMEOUT = 50;
const CLOSE_TIMEOUT = 50;
const JOB_LOOP_ID = 'job.loop';
const SCHEDULE_LOOP_ID = 'schedule.loop';
export class JobService {
private readonly app: AppService;
private readonly queue: string = 'main';
private readonly handlerMap: JobHandlerMap;
private readonly runningConcurrency = new Map<string, number>();
private readonly lock = new AsyncLock();
private readonly sleepScheduler = new SleepScheduler();
private stopped = false;
private runningJobs = 0;
constructor(app: AppService) {
this.app = app;
this.handlerMap = {
'token.delete': new TokenDeleteJobHandler(app),
'account.sync': new AccountSyncJobHandler(app),
'server.sync': new ServerSyncJobHandler(app),
'file.upload': new FileUploadJobHandler(app),
'file.download': new FileDownloadJobHandler(app),
'mutations.sync': new MutationsSyncJobHandler(app),
'temp.files.clean': new TempFilesCleanJobHandler(app),
'workspace.files.clean': new WorkspaceFilesCleanJobHandler(app),
'avatar.download': new AvatarDownloadJobHandler(app),
'avatars.clean': new AvatarsCleanJobHandler(app),
'local.file.download': new LocalFileDownloadJobHandler(app),
};
}
public async init(): Promise<void> {
await this.app.database
.updateTable('jobs')
.set({ status: JobStatus.Waiting })
.execute();
this.jobLoop().catch((err) => console.error('Job loop error:', err));
this.scheduleLoop().catch((err) =>
console.error('Schedule loop error:', err)
);
}
public async addJob(
input: JobInput,
opts?: JobOptions,
scheduleId?: string
): Promise<SelectJob | null> {
const handler = this.handlerMap[input.type] as JobHandler<typeof input>;
if (!handler) {
return null;
}
const now = new Date();
const scheduledAt =
opts?.delay && opts.delay > 0
? new Date(now.getTime() + opts.delay)
: now;
const concurrencyKey = handler.concurrency?.key?.(input);
if (opts?.deduplication) {
const deduplication = opts.deduplication;
const result = await this.lock.acquire(deduplication.key, async () => {
const existing = await this.app.database
.selectFrom('jobs')
.selectAll()
.where('deduplication_key', '=', deduplication.key)
.where('status', '=', JobStatus.Waiting)
.executeTakeFirst();
if (!existing) {
return null;
}
if (!deduplication.replace) {
return existing;
}
const updatedJob = await this.app.database
.updateTable('jobs')
.set({
input: this.toJSON(input),
options: this.toJSON(opts),
scheduled_at: scheduledAt.toISOString(),
concurrency_key: concurrencyKey || null,
updated_at: now.toISOString(),
})
.where('id', '=', existing.id)
.where('status', '=', JobStatus.Waiting)
.returningAll()
.executeTakeFirst();
return updatedJob ?? null;
});
if (result) {
const timestamp = new Date(result.scheduled_at).getTime();
this.sleepScheduler.updateResolveTimeIfEarlier(JOB_LOOP_ID, timestamp);
return result;
}
}
const id = generateId(IdType.Job);
const job = await this.app.database
.insertInto('jobs')
.returningAll()
.values({
id,
queue: this.queue,
input: this.toJSON(input),
options: this.toJSON(opts),
status: JobStatus.Waiting,
retries: 0,
scheduled_at: scheduledAt.toISOString(),
deduplication_key: opts?.deduplication?.key || null,
concurrency_key: concurrencyKey || null,
schedule_id: scheduleId || null,
created_at: now.toISOString(),
updated_at: now.toISOString(),
})
.executeTakeFirst();
if (job) {
const timestamp = new Date(job.scheduled_at).getTime();
this.sleepScheduler.updateResolveTimeIfEarlier(JOB_LOOP_ID, timestamp);
}
return job ?? null;
}
public async upsertJobSchedule(
id: string,
input: JobInput,
interval: number,
opts?: JobScheduleOptions
): Promise<SelectJobSchedule | null> {
const now = new Date().toISOString();
const nextRunAt = new Date(Date.parse(now) + interval).toISOString();
const schedule = await this.app.database
.insertInto('job_schedules')
.returningAll()
.values({
id,
queue: this.queue,
input: this.toJSON(input),
options: this.toJSON(opts),
status: JobScheduleStatus.Active,
interval: interval,
next_run_at: nextRunAt,
created_at: now,
updated_at: now,
})
.onConflict((oc) =>
oc.columns(['id']).doUpdateSet({
input: this.toJSON(input),
options: this.toJSON(opts),
interval: interval,
next_run_at: nextRunAt,
updated_at: now,
status: JobScheduleStatus.Active,
})
)
.executeTakeFirst();
if (schedule) {
const timestamp = new Date(schedule.next_run_at).getTime();
this.sleepScheduler.updateResolveTimeIfEarlier(
SCHEDULE_LOOP_ID,
timestamp
);
}
return schedule ?? null;
}
public async removeJobSchedule(id: string): Promise<void> {
await this.app.database
.deleteFrom('job_schedules')
.where('id', '=', id)
.execute();
}
public async triggerJobSchedule(id: string): Promise<SelectJob | null> {
const schedule = await this.app.database
.selectFrom('job_schedules')
.selectAll()
.where('id', '=', id)
.where('status', '=', JobScheduleStatus.Active)
.executeTakeFirst();
if (!schedule) {
return null;
}
return this.addJobFromSchedule(schedule);
}
public async close() {
this.stopped = true;
while (this.runningJobs > 0) {
await new Promise((r) => setTimeout(r, CLOSE_TIMEOUT));
}
}
private async jobLoop() {
while (!this.stopped) {
if (this.runningJobs >= MAX_CONCURRENCY) {
const timestamp = Date.now() + JOBS_CONCURRENCY_TIMEOUT;
await this.sleepScheduler.sleepUntil(JOB_LOOP_ID, timestamp);
continue;
}
const limitReachedTypes = new Set<string>();
for (const [type, handler] of Object.entries(this.handlerMap)) {
if (handler.concurrency) {
const currentCount = this.runningConcurrency.get(type) || 0;
if (currentCount >= handler.concurrency.limit) {
limitReachedTypes.add(type);
}
}
}
const now = new Date();
const jobRow = await this.app.database
.updateTable('jobs')
.set({
status: JobStatus.Active,
updated_at: now.toISOString(),
})
.where('id', 'in', (qb) =>
qb
.selectFrom('jobs')
.select('id')
.where('queue', '=', this.queue)
.where('status', '=', JobStatus.Waiting)
.where('scheduled_at', '<=', now.toISOString())
.where('concurrency_key', 'not in', Array.from(limitReachedTypes))
.orderBy('scheduled_at', 'asc')
.limit(1)
)
.returningAll()
.executeTakeFirst();
if (!jobRow) {
const nextScheduledJob = await this.app.database
.selectFrom('jobs')
.select('scheduled_at')
.where('queue', '=', this.queue)
.where('status', '=', JobStatus.Waiting)
.orderBy('scheduled_at', 'asc')
.limit(1)
.executeTakeFirst();
if (nextScheduledJob) {
const timestamp = new Date(nextScheduledJob.scheduled_at).getTime();
await this.sleepScheduler.sleepUntil(JOB_LOOP_ID, timestamp);
} else {
const timestamp = new Date(
now.getTime() + JOBS_MAX_TIMEOUT
).getTime();
await this.sleepScheduler.sleepUntil(JOB_LOOP_ID, timestamp);
}
continue;
}
const input = this.fromJSON<JobInput>(jobRow.input) as JobInput;
const handler = this.handlerMap[input.type] as JobHandler<typeof input>;
if (handler?.concurrency) {
const concurrencyKey = handler.concurrency.key(input);
const currentCount = this.runningConcurrency.get(concurrencyKey) || 0;
if (currentCount >= handler.concurrency.limit) {
await this.app.database
.updateTable('jobs')
.set({ status: JobStatus.Waiting, updated_at: now.toISOString() })
.where('id', '=', jobRow.id)
.execute();
continue;
}
}
this.handleJob(jobRow).catch((err) =>
console.error('Job handling error:', err)
);
}
}
private async scheduleLoop() {
while (!this.stopped) {
const now = new Date();
const schedules = await this.app.database
.selectFrom('job_schedules')
.selectAll()
.where('status', '=', JobScheduleStatus.Active)
.where('next_run_at', '<=', now.toISOString())
.execute();
for (const schedule of schedules) {
try {
await this.addJobFromSchedule(schedule);
const nextRunAt = new Date(now.getTime() + schedule.interval);
await this.app.database
.updateTable('job_schedules')
.set({
next_run_at: nextRunAt.toISOString(),
last_run_at: now.toISOString(),
updated_at: now.toISOString(),
})
.where('id', '=', schedule.id)
.execute();
} catch (error) {
console.error(`Error processing schedule ${schedule.id}:`, error);
}
}
const nextSchedule = await this.app.database
.selectFrom('job_schedules')
.select('next_run_at')
.where('status', '=', JobScheduleStatus.Active)
.orderBy('next_run_at', 'asc')
.limit(1)
.executeTakeFirst();
if (nextSchedule) {
const timestamp = new Date(nextSchedule.next_run_at).getTime();
await this.sleepScheduler.sleepUntil(SCHEDULE_LOOP_ID, timestamp);
} else {
const timestamp = new Date(
now.getTime() + SCHEDULES_MAX_TIMEOUT
).getTime();
await this.sleepScheduler.sleepUntil(SCHEDULE_LOOP_ID, timestamp);
}
}
}
private async addJobFromSchedule(schedule: SelectJobSchedule) {
const input = this.fromJSON<JobInput>(schedule.input) as JobInput;
const options = this.fromJSON<JobScheduleOptions>(schedule.options);
return this.addJob(
input,
{
retries: options?.retries,
deduplication: options?.deduplication,
},
schedule.id
);
}
private async handleJob(jobRow: SelectJob) {
const options = this.fromJSON<JobOptions>(jobRow.options) ?? {};
const input = this.fromJSON<JobInput>(jobRow.input) as JobInput;
const job: Job = {
id: jobRow.id,
queue: jobRow.queue,
input,
options,
status: jobRow.status,
retries: jobRow.retries,
deduplicationKey: jobRow.deduplication_key ?? undefined,
concurrencyKey: jobRow.concurrency_key ?? undefined,
createdAt: jobRow.created_at,
updatedAt: jobRow.updated_at,
scheduledAt: jobRow.scheduled_at,
};
try {
this.runningJobs++;
if (jobRow.concurrency_key) {
const current = this.runningConcurrency.get(input.type) || 0;
this.runningConcurrency.set(input.type, current + 1);
}
const handler = this.handlerMap[input.type] as JobHandler<typeof input>;
if (!handler) {
await this.app.database
.deleteFrom('jobs')
.where('id', '=', jobRow.id)
.execute();
return;
}
const output = await handler.handleJob(job.input);
if (output.type === 'retry') {
const retryAt = new Date(Date.now() + output.delay);
await this.app.database
.updateTable('jobs')
.set({
status: JobStatus.Waiting,
scheduled_at: retryAt.toISOString(),
updated_at: new Date().toISOString(),
})
.where('id', '=', jobRow.id)
.where('status', '=', JobStatus.Active)
.execute();
} else if (output.type === 'cancel') {
if (jobRow.schedule_id) {
await this.removeJobSchedule(jobRow.schedule_id);
}
await this.app.database
.deleteFrom('jobs')
.where('id', '=', jobRow.id)
.execute();
} else if (output.type === 'success') {
await this.app.database
.deleteFrom('jobs')
.where('id', '=', jobRow.id)
.execute();
}
} catch (error) {
console.error(`Job ${jobRow.id} failed:`, error);
const retries = jobRow.retries + 1;
if (options.retries && retries < options.retries) {
const retryDelay = Math.min(1000 * Math.pow(2, retries), 60000);
const retryAt = new Date(Date.now() + retryDelay);
await this.app.database
.updateTable('jobs')
.set({
status: JobStatus.Waiting,
retries,
scheduled_at: retryAt.toISOString(),
updated_at: new Date().toISOString(),
})
.where('id', '=', jobRow.id)
.execute();
} else {
await this.app.database
.deleteFrom('jobs')
.where('id', '=', jobRow.id)
.execute();
}
} finally {
this.runningJobs--;
if (jobRow.concurrency_key) {
const current = this.runningConcurrency.get(input.type) || 0;
if (current > 0) {
this.runningConcurrency.set(input.type, current - 1);
}
}
}
}
private toJSON(value: unknown) {
return JSON.stringify(value ?? null);
}
private fromJSON<T>(txt: string | null): T | null {
if (txt == null) return null;
return JSON.parse(txt) as T;
}
}