Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import { randomUUID } from "crypto";
import { getACUCTeam } from "../../controllers/auth";
import { logger as _logger } from "../../lib/logger";
import { getScrapeZDR, getSearchForcedKind } from "../../lib/zdr-helpers";
import type { TeamFlags } from "../../controllers/v2/types";
import { addMonitorCheckJob } from "./queue";
import {
advanceMonitorAfterSkippedCheck,
claimDueMonitors,
createMonitorCheck,
deferMonitorClaim,
dispatchScheduledMonitorCheck,
getMonitorCheckForUpdate,
updateMonitor,
updateMonitorCheck,
updateMonitorCheckIfStatus,
updateMonitorScheduleAfterRun,
} from "./store";
import { autumnService } from "../autumn/autumn.service";
import { orgIdForTeam } from "../../lib/team-org";
import { isMonitorCheckStale, MONITOR_CHECK_STALE_ERROR } from "./stale";
import { validateMonitorCron } from "./cron";
import { monitorJitterOffsetMs } from "./jitter";
import type { MonitorRow } from "./types";
const logger = _logger.child({ module: "monitoring-scheduler" });
export const MONITOR_ZDR_PAUSED_ERROR =
"Monitoring is not supported for zero data retention teams, so this monitor was paused.";
export const MONITOR_ZDR_REFUSED_ERROR =
"Monitoring requires retained snapshots and diffs, and is not supported for zero data retention teams.";
// Search monitors route to a dedicated check queue (see queue.ts).
export function monitorIsSearch(monitor: MonitorRow): boolean {
return (monitor.targets ?? []).some(target => target.type === "search");
}
/** Monitors run without zero data retention, so a team that requires it for scrape, or for search when a target is a search, cannot run one. */
export function monitorRefusedForZdr(
flags: TeamFlags | undefined,
targets: { type: string }[],
): boolean {
return (
getScrapeZDR(flags) === "forced" ||
(getSearchForcedKind(flags) !== null &&
targets.some(target => target.type === "search"))
);
}
export async function enqueueMonitorCheck(params: {
monitorId: string;
checkId: string;
teamId: string;
search?: boolean;
}): Promise<void> {
await addMonitorCheckJob(
{
monitorId: params.monitorId,
checkId: params.checkId,
teamId: params.teamId,
},
{ search: params.search },
);
}
export async function enqueueDueMonitorChecks(
params: {
workerId?: string;
limit?: number;
leaseSeconds?: number;
} = {},
): Promise<number> {
const workerId = params.workerId ?? `monitor-scheduler-${randomUUID()}`;
const monitors = await claimDueMonitors({
workerId,
limit: params.limit ?? 10,
leaseSeconds: params.leaseSeconds ?? 60,
});
let enqueued = 0;
for (let monitor of monitors) {
let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;
let dispatched = false;
try {
if (await deferForJitter(monitor)) continue;
if (monitor.current_check_id) {
const resolution = await clearFinishedOrStaleCurrentCheck(monitor);
if (resolution === "lost_claim") continue;
if (resolution === "cleared") {
monitor = { ...monitor, current_check_id: null };
}
}
// A monitor created before its team required zero data retention would
// otherwise keep running without it.
const acuc = await getACUCTeam(monitor.team_id);
if (
monitorRefusedForZdr(acuc?.flags ?? undefined, monitor.targets ?? [])
) {
const refused = await createMonitorCheck({
monitor,
trigger: "scheduled",
scheduledFor: monitor.next_run_at,
status: "failed",
});
const finished = await updateMonitorCheck(refused.id, {
status: "failed",
finished_at: new Date().toISOString(),
error: MONITOR_ZDR_PAUSED_ERROR,
});
await updateMonitor({
teamId: monitor.team_id,
monitorId: monitor.id,
input: { status: "paused" },
});
// Advanced as active so next_run_at stays set and a resume by status
// alone schedules it again, as for a monitor its owner paused.
await advanceMonitorAfterSkippedCheck({ monitor, check: finished });
logger.warn("Paused monitor: its team requires zero data retention", {
monitorId: monitor.id,
checkId: finished.id,
teamId: monitor.team_id,
});
continue;
}
if (monitor.current_check_id) {
const skipped = await createMonitorCheck({
monitor,
trigger: "scheduled",
scheduledFor: monitor.next_run_at,
status: "skipped_overlap",
});
const finished = await updateMonitorCheck(skipped.id, {
status: "skipped_overlap",
finished_at: new Date().toISOString(),
error: "Previous monitor check is still running.",
});
await advanceMonitorAfterSkippedCheck({ monitor, check: finished });
continue;
}
check = await createMonitorCheck({
monitor,
trigger: "scheduled",
scheduledFor: monitor.next_run_at,
});
dispatched = await dispatchScheduledMonitorCheck({
monitor,
checkId: check.id,
});
if (!dispatched) {
check = await updateMonitorCheck(check.id, {
status: "skipped_overlap",
finished_at: new Date().toISOString(),
error: "Previous monitor check is still running.",
});
await advanceMonitorAfterSkippedCheck({ monitor, check });
continue;
}
await enqueueMonitorCheck({
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
search: monitorIsSearch(monitor),
});
enqueued++;
} catch (error) {
if (check) {
const failed = await updateMonitorCheck(check.id, {
status: "failed",
finished_at: new Date().toISOString(),
error: error instanceof Error ? error.message : String(error),
}).catch(updateError => {
logger.error("Failed to mark monitor check enqueue failure", {
updateError,
error,
monitorId: monitor.id,
checkId: check?.id,
teamId: monitor.team_id,
});
return null;
});
if (failed && dispatched) {
await updateMonitorScheduleAfterRun({
monitor,
check: failed,
}).catch(updateError => {
logger.error("Failed to clear failed dispatched monitor check", {
updateError,
error,
monitorId: monitor.id,
checkId: failed.id,
teamId: monitor.team_id,
});
});
}
}
logger.error("Failed to enqueue due monitor check", {
error,
monitorId: monitor.id,
teamId: monitor.team_id,
});
}
}
return enqueued;
}
async function deferForJitter(monitor: MonitorRow): Promise<boolean> {
if (!monitor.next_run_at) return false;
const { intervalMs } = validateMonitorCron(
monitor.schedule_cron,
monitor.schedule_timezone,
);
const dueAt =
new Date(monitor.next_run_at).getTime() +
monitorJitterOffsetMs(monitor.id, intervalMs);
if (Date.now() >= dueAt) return false;
try {
await deferMonitorClaim(monitor.id, new Date(dueAt));
} catch (error) {
logger.warn("Failed to defer monitor claim for jitter", {
error,
monitorId: monitor.id,
});
return false;
}
return true;
}
async function clearFinishedOrStaleCurrentCheck(
monitor: MonitorRow,
): Promise<"cleared" | "active" | "lost_claim"> {
if (!monitor.current_check_id) return "cleared";
const current = await getMonitorCheckForUpdate(
monitor.team_id,
monitor.id,
monitor.current_check_id,
);
if (!current) return "active";
if (current.status === "running" || current.status === "queued") {
if (!isMonitorCheckStale(current, new Date(), monitor.targets))
return "active";
// Compete with the reconciler before releasing its hold. A failed claim
// means another worker owns completion; leave its billing and schedule alone.
const failed = await updateMonitorCheckIfStatus(
current.id,
current.status,
{
status: "failed",
finished_at: new Date().toISOString(),
actual_credits: 0,
error: MONITOR_CHECK_STALE_ERROR,
},
);
if (!failed) return "lost_claim";
let released = true;
if (failed.autumn_lock_id) {
// The billing service no longer finds the org for itself; without one
// the release goes through firebill when it is configured, settled by
// lock id alone.
const orgId = await orgIdForTeam(monitor.team_id);
released = await autumnService
.finalizeCreditsLock({
lockId: failed.autumn_lock_id,
action: "release",
properties: {
source: "monitorCheck",
endpoint: "monitor",
jobId: failed.id,
},
team: orgId ? { teamId: monitor.team_id, orgId } : undefined,
})
.catch(error => {
logger.warn("Failed to release stale monitor check credit lock", {
error,
monitorId: monitor.id,
checkId: failed.id,
lockId: failed.autumn_lock_id,
});
return false;
});
}
const finalized = await updateMonitorCheck(failed.id, {
billing_status: failed.autumn_lock_id
? released
? "released"
: "failed"
: "not_applicable",
});
await updateMonitorScheduleAfterRun({
monitor,
check: finalized,
});
return "cleared";
}
await updateMonitorScheduleAfterRun({
monitor,
check: current,
});
return "cleared";
}