Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

9.6 KB
Raw Blame
import { randomUUID } from "crypto";
import { getACUCTeam } from "../../controllers/auth";
import { logger as _logger } from "../../lib/logger";
import { getScrapeZDR, getSearchForcedKind } from "../../lib/zdr-helpers";
import type { TeamFlags } from "../../controllers/v2/types";
import { addMonitorCheckJob } from "./queue";
import {
  advanceMonitorAfterSkippedCheck,
  claimDueMonitors,
  createMonitorCheck,
  deferMonitorClaim,
  dispatchScheduledMonitorCheck,
  getMonitorCheckForUpdate,
  updateMonitor,
  updateMonitorCheck,
  updateMonitorCheckIfStatus,
  updateMonitorScheduleAfterRun,
} from "./store";
import { autumnService } from "../autumn/autumn.service";
import { orgIdForTeam } from "../../lib/team-org";
import { isMonitorCheckStale, MONITOR_CHECK_STALE_ERROR } from "./stale";
import { validateMonitorCron } from "./cron";
import { monitorJitterOffsetMs } from "./jitter";
import type { MonitorRow } from "./types";

const logger = _logger.child({ module: "monitoring-scheduler" });

export const MONITOR_ZDR_PAUSED_ERROR =
  "Monitoring is not supported for zero data retention teams, so this monitor was paused.";

export const MONITOR_ZDR_REFUSED_ERROR =
  "Monitoring requires retained snapshots and diffs, and is not supported for zero data retention teams.";

// Search monitors route to a dedicated check queue (see queue.ts).
export function monitorIsSearch(monitor: MonitorRow): boolean {
  return (monitor.targets ?? []).some(target => target.type === "search");
}

/** Monitors run without zero data retention, so a team that requires it for scrape, or for search when a target is a search, cannot run one. */
export function monitorRefusedForZdr(
  flags: TeamFlags | undefined,
  targets: { type: string }[],
): boolean {
  return (
    getScrapeZDR(flags) === "forced" ||
    (getSearchForcedKind(flags) !== null &&
      targets.some(target => target.type === "search"))
  );
}

export async function enqueueMonitorCheck(params: {
  monitorId: string;
  checkId: string;
  teamId: string;
  search?: boolean;
}): Promise<void> {
  await addMonitorCheckJob(
    {
      monitorId: params.monitorId,
      checkId: params.checkId,
      teamId: params.teamId,
    },
    { search: params.search },
  );
}

export async function enqueueDueMonitorChecks(
  params: {
    workerId?: string;
    limit?: number;
    leaseSeconds?: number;
  } = {},
): Promise<number> {
  const workerId = params.workerId ?? `monitor-scheduler-${randomUUID()}`;
  const monitors = await claimDueMonitors({
    workerId,
    limit: params.limit ?? 10,
    leaseSeconds: params.leaseSeconds ?? 60,
  });

  let enqueued = 0;
  for (let monitor of monitors) {
    let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;
    let dispatched = false;
    try {
      if (await deferForJitter(monitor)) continue;

      if (monitor.current_check_id) {
        const resolution = await clearFinishedOrStaleCurrentCheck(monitor);
        if (resolution === "lost_claim") continue;
        if (resolution === "cleared") {
          monitor = { ...monitor, current_check_id: null };
        }
      }

      // A monitor created before its team required zero data retention would
      // otherwise keep running without it.
      const acuc = await getACUCTeam(monitor.team_id);
      if (
        monitorRefusedForZdr(acuc?.flags ?? undefined, monitor.targets ?? [])
      ) {
        const refused = await createMonitorCheck({
          monitor,
          trigger: "scheduled",
          scheduledFor: monitor.next_run_at,
          status: "failed",
        });
        const finished = await updateMonitorCheck(refused.id, {
          status: "failed",
          finished_at: new Date().toISOString(),
          error: MONITOR_ZDR_PAUSED_ERROR,
        });
        await updateMonitor({
          teamId: monitor.team_id,
          monitorId: monitor.id,
          input: { status: "paused" },
        });
        // Advanced as active so next_run_at stays set and a resume by status
        // alone schedules it again, as for a monitor its owner paused.
        await advanceMonitorAfterSkippedCheck({ monitor, check: finished });
        logger.warn("Paused monitor: its team requires zero data retention", {
          monitorId: monitor.id,
          checkId: finished.id,
          teamId: monitor.team_id,
        });
        continue;
      }

      if (monitor.current_check_id) {
        const skipped = await createMonitorCheck({
          monitor,
          trigger: "scheduled",
          scheduledFor: monitor.next_run_at,
          status: "skipped_overlap",
        });
        const finished = await updateMonitorCheck(skipped.id, {
          status: "skipped_overlap",
          finished_at: new Date().toISOString(),
          error: "Previous monitor check is still running.",
        });
        await advanceMonitorAfterSkippedCheck({ monitor, check: finished });
        continue;
      }

      check = await createMonitorCheck({
        monitor,
        trigger: "scheduled",
        scheduledFor: monitor.next_run_at,
      });
      dispatched = await dispatchScheduledMonitorCheck({
        monitor,
        checkId: check.id,
      });
      if (!dispatched) {
        check = await updateMonitorCheck(check.id, {
          status: "skipped_overlap",
          finished_at: new Date().toISOString(),
          error: "Previous monitor check is still running.",
        });
        await advanceMonitorAfterSkippedCheck({ monitor, check });
        continue;
      }

      await enqueueMonitorCheck({
        monitorId: monitor.id,
        checkId: check.id,
        teamId: monitor.team_id,
        search: monitorIsSearch(monitor),
      });
      enqueued++;
    } catch (error) {
      if (check) {
        const failed = await updateMonitorCheck(check.id, {
          status: "failed",
          finished_at: new Date().toISOString(),
          error: error instanceof Error ? error.message : String(error),
        }).catch(updateError => {
          logger.error("Failed to mark monitor check enqueue failure", {
            updateError,
            error,
            monitorId: monitor.id,
            checkId: check?.id,
            teamId: monitor.team_id,
          });
          return null;
        });

        if (failed && dispatched) {
          await updateMonitorScheduleAfterRun({
            monitor,
            check: failed,
          }).catch(updateError => {
            logger.error("Failed to clear failed dispatched monitor check", {
              updateError,
              error,
              monitorId: monitor.id,
              checkId: failed.id,
              teamId: monitor.team_id,
            });
          });
        }
      }

      logger.error("Failed to enqueue due monitor check", {
        error,
        monitorId: monitor.id,
        teamId: monitor.team_id,
      });
    }
  }

  return enqueued;
}

async function deferForJitter(monitor: MonitorRow): Promise<boolean> {
  if (!monitor.next_run_at) return false;
  const { intervalMs } = validateMonitorCron(
    monitor.schedule_cron,
    monitor.schedule_timezone,
  );
  const dueAt =
    new Date(monitor.next_run_at).getTime() +
    monitorJitterOffsetMs(monitor.id, intervalMs);
  if (Date.now() >= dueAt) return false;
  try {
    await deferMonitorClaim(monitor.id, new Date(dueAt));
  } catch (error) {
    logger.warn("Failed to defer monitor claim for jitter", {
      error,
      monitorId: monitor.id,
    });
    return false;
  }
  return true;
}

async function clearFinishedOrStaleCurrentCheck(
  monitor: MonitorRow,
): Promise<"cleared" | "active" | "lost_claim"> {
  if (!monitor.current_check_id) return "cleared";

  const current = await getMonitorCheckForUpdate(
    monitor.team_id,
    monitor.id,
    monitor.current_check_id,
  );
  if (!current) return "active";

  if (current.status === "running" || current.status === "queued") {
    if (!isMonitorCheckStale(current, new Date(), monitor.targets))
      return "active";

    // Compete with the reconciler before releasing its hold. A failed claim
    // means another worker owns completion; leave its billing and schedule alone.
    const failed = await updateMonitorCheckIfStatus(
      current.id,
      current.status,
      {
        status: "failed",
        finished_at: new Date().toISOString(),
        actual_credits: 0,
        error: MONITOR_CHECK_STALE_ERROR,
      },
    );
    if (!failed) return "lost_claim";

    let released = true;
    if (failed.autumn_lock_id) {
      // The billing service no longer finds the org for itself; without one
      // the release goes through firebill when it is configured, settled by
      // lock id alone.
      const orgId = await orgIdForTeam(monitor.team_id);
      released = await autumnService
        .finalizeCreditsLock({
          lockId: failed.autumn_lock_id,
          action: "release",
          properties: {
            source: "monitorCheck",
            endpoint: "monitor",
            jobId: failed.id,
          },
          team: orgId ? { teamId: monitor.team_id, orgId } : undefined,
        })
        .catch(error => {
          logger.warn("Failed to release stale monitor check credit lock", {
            error,
            monitorId: monitor.id,
            checkId: failed.id,
            lockId: failed.autumn_lock_id,
          });
          return false;
        });
    }

    const finalized = await updateMonitorCheck(failed.id, {
      billing_status: failed.autumn_lock_id
        ? released
          ? "released"
          : "failed"
        : "not_applicable",
    });
    await updateMonitorScheduleAfterRun({
      monitor,
      check: finalized,
    });
    return "cleared";
  }

  await updateMonitorScheduleAfterRun({
    monitor,
    check: current,
  });
  return "cleared";
}