A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.
// SPDX-License-Identifier: AGPL-3.0-or-later
import {currentActivityMeta} from '@app/api/infrastructure/activity/ActivityMeta';
import {
ACTIVITY_SPOOL_KEY,
type ActivityDrainResult,
type ActivityPublisher,
type ActivitySpoolEntry,
drainActivitySpool,
spoolActivity,
} from '@app/api/infrastructure/activity/ActivitySpool';
import {
type Body,
EVENT_CLASS,
EVENT_MAJOR,
EVENT_TTL,
EVENTS_STREAM,
type EventKind,
eventSubject,
HEADER,
type Meta,
} from '@app/api/infrastructure/activity/Contract.generated';
import {Logger} from '@app/api/Logger';
import {createSnowflake} from '@fluxer/snowflake/src/Snowflake';
import {JetStreamApiError, type JetStreamClient, type JetStreamManager} from '@nats-io/jetstream';
import {headers} from '@nats-io/transport-node';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
export type ActivityData<K extends EventKind> = Extract<Body, {kind: K}>['data'];
type ActivityResult = 'published' | 'spooled' | 'dropped' | 'rejected';
type ActivityKv = Pick<IKVProvider, 'rpush' | 'llen' | 'lrange' | 'ltrim'>;
const PUBLISH_TIMEOUT_MS = 1000;
const RETRY_DELAYS_MS = [100, 400];
const QUEUE_CAPACITY = 20_000;
const QUEUE_CONCURRENCY = 64;
const STREAM_PROBE_INTERVAL_MS = 15_000;
const STREAM_IDLE_PROBE_INTERVAL_MS = 300_000;
const STREAM_REPROBE_INTERVAL_MS = 2_000;
const STREAM_NOT_FOUND = 10059;
const SPOOL_DEPTH_INTERVAL_MS = 15_000;
interface PendingActivity {
entry: ActivitySpoolEntry;
always: boolean;
}
type StreamState = 'present' | 'missing' | 'unknown';
type EmitMode = 'publish' | 'spool' | 'off';
interface ActivityRuntime {
publisher: ActivityPublisher;
kv: ActivityKv;
mode: () => EmitMode;
stop: () => void;
}
let runtime: ActivityRuntime | null = null;
const queue: Array<PendingActivity> = [];
let inFlight = 0;
let idleWaiters: Array<() => void> = [];
const counts = new Map<string, number>();
const drainCounts = new Map<string, number>();
let spoolDepth = 0;
function delay(ms: number): Promise<void> {
return new Promise((resolve) => {
setTimeout(resolve, ms).unref?.();
});
}
function count(kind: EventKind, result: ActivityResult): void {
const key = `kind="${kind}",result="${result}"`;
counts.set(key, (counts.get(key) ?? 0) + 1);
}
function isRejection(error: unknown): boolean {
return error instanceof JetStreamApiError || (error instanceof Error && error.name === 'JetStreamNotEnabled');
}
export function jetStreamActivityPublisher(js: JetStreamClient): ActivityPublisher {
return {
async publish(subject, payload, options) {
const h = headers();
if (options.spooled) h.set(HEADER.spooled, '1');
await js.publish(subject, payload, {
msgID: options.msgID,
ttl: options.ttl,
timeout: PUBLISH_TIMEOUT_MS,
headers: h,
});
},
};
}
function freshActivityId(nowMs = Date.now()): string {
return createSnowflake({
timestamp: nowMs,
workerId: Math.floor(Math.random() * 1024),
sequence: Math.floor(Math.random() * 4096),
}).toString();
}
async function publishWithRetry(publisher: ActivityPublisher, entry: ActivitySpoolEntry): Promise<ActivityResult> {
for (let attempt = 0; ; attempt++) {
try {
await publisher.publish(entry.subject, entry.data, {msgID: entry.id, ttl: entry.ttl, spooled: false});
return 'published';
} catch (error) {
if (isRejection(error)) {
Logger.warn({err: error, kind: entry.kind}, 'Activity event publish rejected');
return 'rejected';
}
const wait = RETRY_DELAYS_MS[attempt];
if (wait === undefined) {
Logger.warn({err: error, kind: entry.kind}, 'Activity event publish failed');
return 'spooled';
}
await delay(wait);
}
}
}
async function spool(kv: ActivityKv, entry: ActivitySpoolEntry, always: boolean): Promise<boolean> {
try {
const outcome = await spoolActivity(kv, entry, always);
if (outcome === 'dropped') {
count(entry.kind, 'dropped');
return false;
}
spoolDepth++;
return true;
} catch (error) {
Logger.error({err: error, kind: entry.kind, id: entry.id}, 'Activity event could not be spooled');
count(entry.kind, 'dropped');
return false;
}
}
async function deliver(item: PendingActivity): Promise<void> {
const active = runtime;
if (!active) {
count(item.entry.kind, 'dropped');
return;
}
const result = await publishWithRetry(active.publisher, item.entry);
if (result === 'published') {
count(item.entry.kind, 'published');
return;
}
if (result === 'rejected') count(item.entry.kind, 'rejected');
if ((await spool(active.kv, item.entry, item.always)) && result === 'spooled') {
count(item.entry.kind, 'spooled');
}
}
function pump(): void {
while (inFlight < QUEUE_CONCURRENCY && queue.length > 0) {
const item = queue.shift()!;
inFlight++;
void deliver(item).finally(() => {
inFlight--;
pump();
});
}
if (inFlight === 0 && queue.length === 0) {
const waiters = idleWaiters;
idleWaiters = [];
for (const resolve of waiters) resolve();
}
}
async function buildEntry<K extends EventKind>(
kind: K,
key: string,
data: ActivityData<K>,
meta: Meta | null,
naturalId: string | null,
atMs: number,
): Promise<ActivitySpoolEntry> {
const id = `${kind}:${naturalId ?? freshActivityId(atMs)}`;
const event = {
v: EVENT_MAJOR[kind],
id,
at_ms: atMs,
key,
meta: meta ?? (await currentActivityMeta()),
kind,
data,
};
return {kind, subject: eventSubject(kind, key), id, ttl: EVENT_TTL[kind], at_ms: atMs, data: JSON.stringify(event)};
}
export async function emitActivity<K extends EventKind>(
kind: K,
key: string,
data: ActivityData<K>,
meta: Meta | null = null,
naturalId: string | null = null,
): Promise<void> {
const active = runtime;
if (!active) return;
const mode = active.mode();
if (mode === 'off') {
count(kind, 'dropped');
return;
}
const fact = EVENT_CLASS[kind] === 'fact';
try {
const entry = await buildEntry(kind, key, data, meta, naturalId, Date.now());
if (mode === 'spool') {
if (await spool(active.kv, entry, fact)) count(kind, 'rejected');
return;
}
if (fact) {
await deliver({entry, always: true});
return;
}
if (queue.length >= QUEUE_CAPACITY) {
if (runtime && (await spool(runtime.kv, entry, false))) count(kind, 'spooled');
return;
}
queue.push({entry, always: false});
pump();
} catch (error) {
Logger.error({err: error, kind}, 'Activity event could not be built');
count(kind, 'dropped');
}
}
export function activityEventsPublishing(): boolean {
return runtime?.mode() === 'publish';
}
export function idleActivityEvents(): Promise<void> {
if (inFlight === 0 && queue.length === 0) return Promise.resolve();
return new Promise((resolve) => {
idleWaiters.push(resolve);
});
}
export async function drainActivitySpoolNow(
options: {budgetMs?: number; keepLock?: () => Promise<boolean>} = {},
): Promise<ActivityDrainResult | null> {
const active = runtime;
if (active?.mode() !== 'publish') return null;
const result = await drainActivitySpool(active.publisher, active.kv, options);
spoolDepth = result.remaining;
for (const [name, value] of [
['published', result.published],
['expired', result.expired],
['failed', result.failed],
] as const) {
if (value > 0) drainCounts.set(name, (drainCounts.get(name) ?? 0) + value);
}
return result;
}
export interface ActivityEventsOptions {
publisher: ActivityPublisher;
kv: ActivityKv;
jsm?: Pick<JetStreamManager, 'streams'> | null;
spoolWhileMissing?: boolean;
}
async function probeStream(jsm: Pick<JetStreamManager, 'streams'>): Promise<StreamState> {
try {
await jsm.streams.info(EVENTS_STREAM);
return 'present';
} catch (error) {
if (error instanceof JetStreamApiError && error.code === STREAM_NOT_FOUND) return 'missing';
Logger.warn({err: error}, 'Activity stream probe failed');
return 'unknown';
}
}
export async function startActivityEvents(options: ActivityEventsOptions): Promise<void> {
stopActivityEvents();
const {jsm} = options;
let state: StreamState = jsm ? await probeStream(jsm) : 'present';
const timers: Array<ReturnType<typeof setTimeout>> = [];
const whenMissing: EmitMode = options.spoolWhileMissing ? 'spool' : 'off';
const missingProbeMs = options.spoolWhileMissing ? STREAM_PROBE_INTERVAL_MS : STREAM_IDLE_PROBE_INTERVAL_MS;
let stopped = false;
const mode = (): EmitMode => (state === 'missing' ? whenMissing : 'publish');
const schedule = (probe: Pick<JetStreamManager, 'streams'>) => {
if (stopped || state === 'present') return;
const wait = state === 'unknown' ? STREAM_REPROBE_INTERVAL_MS : missingProbeMs;
const timer = setTimeout(() => {
void probeStream(probe).then((next) => {
if (next !== state) Logger.info({stream: EVENTS_STREAM, state: next}, 'Activity stream state changed');
state = next;
schedule(probe);
});
}, wait);
timer.unref?.();
timers.push(timer);
};
if (jsm && state !== 'present') {
Logger.info({stream: EVENTS_STREAM, state}, 'Activity stream not confirmed, probing until it exists');
schedule(jsm);
}
const depth = setInterval(() => {
if (mode() === 'off') return;
void options.kv
.llen(ACTIVITY_SPOOL_KEY)
.then((value) => {
spoolDepth = value;
})
.catch(() => undefined);
}, SPOOL_DEPTH_INTERVAL_MS);
depth.unref?.();
timers.push(depth);
runtime = {
publisher: options.publisher,
kv: options.kv,
mode,
stop: () => {
stopped = true;
for (const timer of timers) {
clearTimeout(timer);
clearInterval(timer);
}
},
};
}
export async function shutdownActivityEvents(timeoutMs = 2000): Promise<void> {
await Promise.race([idleActivityEvents(), delay(timeoutMs)]);
stopActivityEvents();
}
function stopActivityEvents(): void {
runtime?.stop();
runtime = null;
}
export function renderActivityMetrics(): string {
const lines = [
'# HELP fluxer_api_activity_events_total Activity events by kind and result',
'# TYPE fluxer_api_activity_events_total counter',
...[...counts].map(([labels, value]) => `fluxer_api_activity_events_total{${labels}} ${value}`),
'# HELP fluxer_api_activity_spool_depth Activity events waiting in the spool',
'# TYPE fluxer_api_activity_spool_depth gauge',
`fluxer_api_activity_spool_depth ${spoolDepth}`,
'# HELP fluxer_api_activity_spool_drained_total Spooled activity events by drain result',
'# TYPE fluxer_api_activity_spool_drained_total counter',
...[...drainCounts].map(
([result, value]) => `fluxer_api_activity_spool_drained_total{result="${result}"} ${value}`,
),
];
return lines.join('\n');
}
export function resetActivityEventsForTests(): void {
stopActivityEvents();
queue.length = 0;
inFlight = 0;
idleWaiters = [];
counts.clear();
drainCounts.clear();
spoolDepth = 0;
}
export function activityCountForTests(kind: EventKind, result: ActivityResult): number {
return counts.get(`kind="${kind}",result="${result}"`) ?? 0;
}