A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import {
BatchBuilder,
deleteOneOrMany,
fetchMany,
fetchManyInChunks,
fetchOne,
fetchPage,
type PagedQueryResult,
upsertOne,
} from '@app/api/database/CassandraQueryExecution';
import {Db} from '@app/api/database/CassandraTypes';
import type {JobActiveRow, JobByDayBucketRow, JobByIdRow, JobStatus} from '@app/api/database/types/JobLedgerTypes';
import {JobsActive, JobsByDayBucket, JobsById} from '@app/api/Tables';
import {awaitAll} from '@app/api/utils/ConcurrencyUtils';
import {JOBS_STREAM_MAX_AGE_MS} from '@app/api/worker/JetStreamWorkerQueue';
import {snowflakeToDate} from '@fluxer/snowflake/src/Snowflake';
import {ms, seconds} from 'itty-time';
export interface CreateJobInput {
jobId: bigint;
taskType: string;
payload: Record<string, unknown>;
requestedByUserId: bigint | null;
auditLogReason: string | null;
maxAttempts: number;
runAt: Date | null;
jetStreamLane: string | null;
jetStreamSeq: string | null;
}
export interface ListJobsCursor {
bucketDay: string;
createdAt: Date;
jobId: bigint;
}
export interface ListJobsFilters {
status?: JobStatus | null;
taskType?: string | null;
requestedByUserId?: bigint | null;
}
export interface ListJobsResult {
jobs: Array<JobByIdRow>;
nextCursor: ListJobsCursor | null;
}
export const JOB_LEDGER_TTL_SECONDS = seconds('90 days');
export const JOB_STALE_AFTER_MS = JOBS_STREAM_MAX_AGE_MS + ms('1 day');
export const EXPIRED_JOB_ERROR = 'Expired from the job queue';
const JOB_LEDGER_RETENTION_DAYS = JOB_LEDGER_TTL_SECONDS / seconds('1 day');
const NEWEST_FIRST = {col: 'created_at', direction: 'DESC'} as const;
const FILTERED_SCAN_PAGE_SIZE = 200;
const FETCH_JOB_BY_ID_QUERY = JobsById.select({
where: JobsById.where.eq('job_id'),
});
const FETCH_JOBS_BY_IDS_QUERY = JobsById.select({
where: JobsById.where.in('job_id', 'job_ids'),
});
const ACTIVE_JOBS_QUERY = JobsActive.select();
const ACTIVE_JOB_IDS_QUERY = JobsActive.select({columns: ['job_id']});
const JOBS_AFTER_IN_TIE_QUERY = JobsByDayBucket.select({
where: [
JobsByDayBucket.where.eq('bucket_day'),
JobsByDayBucket.where.eq('created_at'),
JobsByDayBucket.where.lt('job_id'),
],
orderBy: NEWEST_FIRST,
});
type LedgerPosition = Pick<ListJobsCursor, 'createdAt' | 'jobId'>;
function bucketDayFor(d: Date): string {
return d.toISOString().slice(0, 10);
}
function previousBucketDay(day: string): string {
const date = new Date(`${day}T00:00:00Z`);
date.setUTCDate(date.getUTCDate() - 1);
return bucketDayFor(date);
}
function isCompleteJob(row: JobByIdRow | null | undefined): row is JobByIdRow {
return Boolean(row?.created_at && row.task_type && row.status);
}
function dayJobsQuery(olderThanPosition: boolean, limit: number) {
return JobsByDayBucket.select({
where: olderThanPosition
? [JobsByDayBucket.where.eq('bucket_day'), JobsByDayBucket.where.lt('created_at')]
: JobsByDayBucket.where.eq('bucket_day'),
orderBy: NEWEST_FIRST,
limit,
});
}
async function fetchDayAfter(
bucketDay: string,
after: LedgerPosition | null,
limit: number,
): Promise<{rows: Array<JobByDayBucketRow>; exhausted: boolean}> {
if (after === null) {
const rows = await fetchMany<JobByDayBucketRow>(dayJobsQuery(false, limit).bind({bucket_day: bucketDay}));
return {rows, exhausted: rows.length < limit};
}
const ties = await fetchMany<JobByDayBucketRow>(
JOBS_AFTER_IN_TIE_QUERY.bind({bucket_day: bucketDay, created_at: after.createdAt, job_id: after.jobId}),
);
const older = await fetchMany<JobByDayBucketRow>(
dayJobsQuery(true, limit).bind({bucket_day: bucketDay, created_at: after.createdAt}),
);
return {rows: [...ties, ...older], exhausted: older.length < limit};
}
export class JobLedgerRepository {
async createJob(input: CreateJobInput): Promise<Date> {
const now = new Date();
const status: JobStatus = 'queued';
const idRow: JobByIdRow = {
job_id: input.jobId,
task_type: input.taskType,
status,
progress_current: null,
progress_total: null,
progress_message: null,
payload: JSON.stringify(input.payload),
result: null,
error_message: null,
created_at: now,
started_at: null,
completed_at: null,
requested_by_user_id: input.requestedByUserId,
audit_log_reason: input.auditLogReason,
jet_stream_seq: input.jetStreamSeq,
jet_stream_lane: input.jetStreamLane,
attempts: 0,
max_attempts: input.maxAttempts,
run_at: input.runAt,
cancel_requested: false,
context_link: null,
};
const bucketRow: JobByDayBucketRow = {
bucket_day: bucketDayFor(now),
created_at: now,
job_id: input.jobId,
task_type: input.taskType,
status,
requested_by_user_id: input.requestedByUserId,
};
const activeRow: JobActiveRow = {
job_id: input.jobId,
task_type: input.taskType,
status,
requested_by_user_id: input.requestedByUserId,
created_at: now,
started_at: null,
};
const batch = new BatchBuilder();
batch.addPrepared(JobsById.insertWithTtl(idRow, JOB_LEDGER_TTL_SECONDS));
batch.addPrepared(JobsByDayBucket.insertWithTtl(bucketRow, JOB_LEDGER_TTL_SECONDS));
batch.addPrepared(JobsActive.insertWithTtl(activeRow, JOB_LEDGER_TTL_SECONDS));
await batch.executeChunked(10, false);
return now;
}
async getJob(jobId: bigint): Promise<JobByIdRow | null> {
const row = await fetchOne<JobByIdRow>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
return isCompleteJob(row) ? row : null;
}
private async getJobsByIds(jobIds: Array<bigint>): Promise<Map<bigint, JobByIdRow>> {
const rows = await fetchManyInChunks<JobByIdRow, bigint>(FETCH_JOBS_BY_IDS_QUERY, jobIds, (chunk) => ({
job_ids: chunk,
}));
return new Map(rows.filter(isCompleteJob).map((row) => [row.job_id, row]));
}
async discardJob(jobId: bigint, createdAt: Date): Promise<void> {
await awaitAll(
[
deleteOneOrMany(
JobsByDayBucket.deleteByPk({bucket_day: bucketDayFor(createdAt), created_at: createdAt, job_id: jobId}),
),
deleteOneOrMany(JobsById.deleteByPk({job_id: jobId})),
deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId})),
],
'Ledger discard left rows behind',
);
}
async markRunning(jobId: bigint, lane: string): Promise<void> {
const startedAt = new Date();
const status: JobStatus = 'running';
await upsertOne(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), started_at: Db.set(startedAt), jet_stream_lane: Db.set(lane)},
JOB_LEDGER_TTL_SECONDS,
),
);
await upsertOne(
JobsActive.patchByPkWithTtl(
{job_id: jobId},
{status: Db.set(status), started_at: Db.set(startedAt)},
JOB_LEDGER_TTL_SECONDS,
),
);
}
private async markTerminal(
jobId: bigint,
status: Extract<JobStatus, 'succeeded' | 'cancelled' | 'deadletter'>,
extra: Parameters<typeof JobsById.patchByPkWithTtl>[1] = {},
): Promise<void> {
await upsertOne(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{...extra, status: Db.set(status), completed_at: Db.set(new Date())},
JOB_LEDGER_TTL_SECONDS,
),
);
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
}
async markSucceeded(jobId: bigint, result: Record<string, unknown> | null): Promise<void> {
return this.markTerminal(jobId, 'succeeded', {
result: result === null ? Db.clear() : Db.set(JSON.stringify(result)),
});
}
async markCancelled(jobId: bigint): Promise<void> {
return this.markTerminal(jobId, 'cancelled');
}
async markDeadletter(jobId: bigint, errorMessage: string): Promise<void> {
return this.markTerminal(jobId, 'deadletter', {error_message: Db.set(errorMessage)});
}
async reportProgress(jobId: bigint, current: number, total: number | null, message: string | null): Promise<void> {
await upsertOne(
JobsById.patchByPkWithTtl(
{job_id: jobId},
{
progress_current: Db.set(BigInt(current)),
progress_total: total === null ? Db.clear() : Db.set(BigInt(total)),
progress_message: message === null ? Db.clear() : Db.set(message),
},
JOB_LEDGER_TTL_SECONDS,
),
);
}
async setContextLink(jobId: bigint, link: string): Promise<void> {
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {context_link: Db.set(link)}, JOB_LEDGER_TTL_SECONDS));
}
async setJetStreamSeq(jobId: bigint, seq: string): Promise<void> {
await upsertOne(JobsById.patchByPkWithTtl({job_id: jobId}, {jet_stream_seq: Db.set(seq)}, JOB_LEDGER_TTL_SECONDS));
}
async requestCancel(jobId: bigint): Promise<void> {
await upsertOne(
JobsById.patchByPkWithTtl({job_id: jobId}, {cancel_requested: Db.set(true)}, JOB_LEDGER_TTL_SECONDS),
);
}
async isCancelRequested(jobId: bigint): Promise<boolean> {
const row = await fetchOne<{
cancel_requested: boolean | null;
}>(FETCH_JOB_BY_ID_QUERY.bind({job_id: jobId}));
return row?.cancel_requested === true;
}
async incrementAttempts(jobId: bigint): Promise<void> {
const row = await this.getJob(jobId);
if (!row) return;
await upsertOne(
JobsById.patchByPkWithTtl({job_id: jobId}, {attempts: Db.set(row.attempts + 1)}, JOB_LEDGER_TTL_SECONDS),
);
}
async listJobs(opts: {
limit: number;
cursor: ListJobsCursor | null;
filters: ListJobsFilters;
maxLookbackDays: number;
}): Promise<ListJobsResult> {
const {limit, cursor, filters} = opts;
const now = Date.now();
const lookbackDays = Math.min(opts.maxLookbackDays, JOB_LEDGER_RETENTION_DAYS);
const oldestDay = bucketDayFor(new Date(now - lookbackDays * ms('1 day')));
const requestedBy = filters.requestedByUserId ?? null;
const filtered = Boolean(filters.status || filters.taskType || requestedBy !== null);
const jobs: Array<JobByIdRow> = [];
let lastRow: JobByDayBucketRow | null = null;
let day = bucketDayFor(new Date(Math.min(cursor ? cursor.createdAt.getTime() : now, now)));
let after: LedgerPosition | null = cursor;
while (jobs.length < limit && day >= oldestDay) {
const {rows, exhausted} = await fetchDayAfter(
day,
after,
filtered ? FILTERED_SCAN_PAGE_SIZE : limit - jobs.length,
);
const jobsById = await this.getJobsByIds(
rows
.filter(
(row) =>
(!filters.taskType || row.task_type === filters.taskType) &&
(requestedBy === null || row.requested_by_user_id === requestedBy),
)
.map((row) => row.job_id),
);
for (const row of rows) {
after = {createdAt: row.created_at, jobId: row.job_id};
const job = jobsById.get(row.job_id);
if (!job || (filters.status && job.status !== filters.status)) continue;
jobs.push(job);
lastRow = row;
if (jobs.length === limit) break;
}
if (exhausted) {
day = previousBucketDay(day);
after = null;
}
}
return {
jobs,
nextCursor:
lastRow && jobs.length === limit
? {bucketDay: lastRow.bucket_day, createdAt: lastRow.created_at, jobId: lastRow.job_id}
: null,
};
}
async expireStaleActiveJobs(opts: {
staleBeforeMs: number;
pageSize: number;
maxCleared: number;
}): Promise<{cleared: number; expired: number; complete: boolean}> {
let cleared = 0;
let expired = 0;
let pageState: string | null = null;
do {
const page: PagedQueryResult<Pick<JobActiveRow, 'job_id'>> = await fetchPage(
ACTIVE_JOB_IDS_QUERY.bind({}),
undefined,
{
pageSize: opts.pageSize,
pageState,
},
);
for (const {job_id: jobId} of page.rows) {
if (snowflakeToDate(jobId).getTime() >= opts.staleBeforeMs) continue;
if (cleared >= opts.maxCleared) return {cleared, expired, complete: false};
const job = await this.getJob(jobId);
if (job?.status === 'queued' || job?.status === 'running') {
await this.markDeadletter(jobId, EXPIRED_JOB_ERROR);
expired += 1;
} else {
await deleteOneOrMany(JobsActive.deleteByPk({job_id: jobId}));
}
cleared += 1;
}
pageState = page.pageState;
} while (pageState !== null);
return {cleared, expired, complete: true};
}
async listActiveJobs(): Promise<Array<JobByIdRow>> {
const activeRows = await fetchMany<JobActiveRow>(ACTIVE_JOBS_QUERY.bind({}));
const fullRows = await Promise.all(activeRows.map((r) => this.getJob(r.job_id)));
return fullRows.filter((r): r is JobByIdRow => r !== null);
}
}