Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

61.4 KB
Raw Blame
import { v7 as uuidv7 } from "uuid";
import { config } from "../../config";
import { logger as _logger } from "../../lib/logger";
import { logRequest } from "../logging/log_job";
import { getMonitorDiffArtifact } from "../../lib/gcs-monitoring";
import { processJobInternal } from "../worker/scrape-worker";
import {
  NuQJob,
  crawlGroup,
  scrapeQueue,
  resolveNewGroupBackend,
} from "../worker/nuq-router";
import { ScrapeJobData } from "../../types";
import { includesFormat } from "../../lib/format-utils";
import { normalizeMonitorFormats } from "./diff";
import { autumnService } from "../autumn/autumn.service";
import { getBillingQueue } from "../queue-service";
import {
  crawlToCrawler,
  markCrawlActive,
  saveCrawl,
  StoredCrawl,
} from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ, addScrapeJob } from "../queue-jobs";
import {
  CrawlRequest,
  type ScrapeOptions,
  crawlRequestSchema,
  scrapeRequestSchema,
  toV0CrawlerOptions,
} from "../../controllers/v2/types";
import { createWebhookSender, WebhookEvent } from "../webhook";
import { sendMonitorPageWebhook } from "./results";
import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
import { sendMonitoringSlackSummary } from "../notification/monitoring_slack";
import { recordMonitorInAppNotification } from "../notification/monitoring_in_app";
import {
  bulkUpsertMonitorPages,
  calculateMonitorCheckActualCredits,
  countRecentConsecutiveSkippedForCredits,
  getMonitorCheckForUpdate,
  getMonitorForUpdate,
  countMonitorCheckPages,
  insertMonitorCheckPages,
  deleteMonitorCheckPages,
  listActiveMonitorPages,
  listMonitorCheckPages,
  listRunningMonitorChecks,
  markMonitorRunning,
  pauseMonitor,
  updateMonitorCheck,
  updateMonitorCheckIfRunning,
  updateMonitorCheckIfStatus,
  updateMonitorScheduleAfterRun,
  upsertMonitorPage,
} from "./store";
import type {
  MonitorCheckPageInsert,
  MonitorCheckRow,
  MonitorRow,
  MonitorTarget,
} from "./types";
import { withMarkdownFormat } from "./types";
import { redisEvictConnection } from "../redis";
import type { MonitorCheckJobData } from "./queue";
import {
  MONITOR_CHECK_STALE_ERROR,
  isMonitorCheckStale,
  MONITOR_CHECK_STALE_TIMEOUT_MS,
  monitorCheckStaleTimeoutMs,
} from "./stale";
import { trackMonitorCheckStartedInterest } from "./interest";
import { runSearchTarget, type ScrapeSearchResult } from "./search/run";
import { verdictJsonSchema } from "./search/judge";
import { monitorTelemetryMetadata } from "./search/tuning";
import { computeGoalVersion } from "./search/dedupe";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { getACUCTeam } from "../../controllers/auth";
import { isUsOnly } from "../../lib/us-only";
import type { TeamFlags } from "../../controllers/v1/types";
import { orgIdForTeam } from "../../lib/team-org";
import {
  reconstructKnownState,
  searchStatusToPageStatus,
} from "./search/persist";
import { requestCreditsShards } from "../../lib/request-credits-store";

const logger = _logger.child({ module: "monitoring-runner" });
export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };

const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;
const MONITOR_CHECK_PAGE_SCAN_LIMIT = 100_000;
const MONITOR_CHECK_NO_CREDITS_ERROR =
  "Monitor check skipped: insufficient credits.";
const MONITOR_CHECK_REVOKED_ERROR =
  "Monitor check skipped: the partner has revoked this job.";

/**
 * Consecutive credit-skipped checks, this one included, before a `job_revoked`
 * denial stops the schedule. A 410 is permanent, but pausing on one response
 * would let a partner's bad build take down every monitor they fund. Never
 * reached by a 402: the current denial must itself be `job_revoked`.
 */
const MONITOR_GATE_REVOKED_STREAK = 3;

async function claimMonitorNotification(checkId: string): Promise<boolean> {
  const result = await redisEvictConnection.set(
    `monitor-check-notify:${checkId}`,
    "1",
    "EX",
    MONITOR_NOTIFY_CLAIM_TTL_SECONDS,
    "NX",
  );
  return result === "OK";
}

type PageResult = MonitorCheckPageInsert & {
  emailStatus?: string;
};

type MonitorTargetRun =
  | {
      targetId: string;
      type: "scrape";
      expectedJobs: string[];
    }
  | {
      targetId: string;
      type: "crawl";
      crawlId: string;
    }
  | {
      targetId: string;
      type: "search";
      // Set only after the inline search stamps credits; reconciler waits on this so it never finalizes with credits at 0.
      searchCompleted?: boolean;
      resultCount?: number;
      matches?: number;
      summary?: string;
      judgeDegraded?: boolean;
      degradedReason?: string | null;
      searchCredits?: number;
      judgeCredits?: number;
      resultsJudged?: number;
    };

function createMonitorTargetRun(target: MonitorTarget): MonitorTargetRun {
  if (target.type === "scrape") {
    return {
      targetId: target.id,
      type: "scrape",
      expectedJobs: target.urls.map(() => uuidv7()),
    };
  }

  if (target.type === "search") {
    return {
      targetId: target.id,
      type: "search",
    };
  }

  return {
    targetId: target.id,
    type: "crawl",
    crawlId: uuidv7(),
  };
}

async function recoverTargetRunsFromRecordedPages(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
}): Promise<MonitorTargetRun[]> {
  const scrapeRuns: MonitorTargetRun[] = params.monitor.targets
    .filter((target): target is Extract<MonitorTarget, { type: "scrape" }> => {
      return target.type === "scrape";
    })
    .map(target => ({
      targetId: target.id,
      type: "scrape" as const,
      expectedJobs: target.urls.map(
        (_, index) => `recovered:${target.id}:${index}`,
      ),
    }));
  const crawlTargets = params.monitor.targets.filter(
    target => target.type === "crawl",
  );
  if (crawlTargets.length === 0) return scrapeRuns;

  const pages = await listMonitorCheckPages({
    teamId: params.monitor.team_id,
    monitorId: params.monitor.id,
    checkId: params.check.id,
    limit: MONITOR_CHECK_PAGE_SCAN_LIMIT,
    skip: 0,
  });
  const recovered = [...scrapeRuns];

  for (const target of crawlTargets) {
    const page = pages.find(
      candidate =>
        candidate.target_id === target.id &&
        typeof candidate.current_scrape_id === "string",
    );
    if (!page?.current_scrape_id) continue;

    const scrapeJob = await scrapeQueue
      .getJob(page.current_scrape_id, logger)
      .catch(error => {
        logger.warn(
          "Failed to recover monitor crawl target run from page job",
          {
            error,
            monitorId: params.monitor.id,
            checkId: params.check.id,
            targetId: target.id,
            scrapeId: page.current_scrape_id,
          },
        );
        return null;
      });
    const crawlId =
      scrapeJob?.data?.mode === "single_urls"
        ? (scrapeJob.data.crawl_id ?? scrapeJob.groupId)
        : scrapeJob?.groupId;
    if (!crawlId) continue;

    recovered.push({
      targetId: target.id,
      type: "crawl",
      crawlId,
    });
  }

  const recoveredTargetIds = new Set(recovered.map(target => target.targetId));
  if (
    !params.monitor.targets.every(target => recoveredTargetIds.has(target.id))
  ) {
    return [];
  }

  return recovered;
}

function withMonitorScrapeDefaults(
  options: Record<string, unknown>,
): ScrapeOptions {
  const formats = Array.isArray(options.formats)
    ? normalizeMonitorFormats(options.formats)
    : options.formats;
  return {
    maxAge: 0,
    ...withMarkdownFormat({ ...options, formats }),
  };
}

export function estimateActualCredits(doc: any, options?: any): number {
  // Prefer the credits the scrape path actually recorded when present.
  const creditsUsed = doc?.metadata?.creditsUsed;
  if (typeof creditsUsed === "number" && Number.isFinite(creditsUsed)) {
    return creditsUsed;
  }
  const formats = Array.isArray(options?.formats) ? options.formats : [];
  // Only charge the JSON-extraction premium when extraction produced a json; a
  // failed extraction still scraped the page, so fall back to base credit.
  // Deterministic JSON costs 7 (reusable extractor); plain JSON 5.
  const producedJson = doc?.json != null;
  if (!producedJson) return 1;
  if (includesFormat(formats, "deterministicJson")) return 7;
  if (includesFormat(formats, "json")) return 5;
  return 1;
}

// Deep-mode search-monitor page scrape, inline (skipNuq) so it bypasses scrape
// concurrency; caller bounds fan-out via SEARCH_SCRAPE_CONCURRENCY. Never billed
// per-page — search monitors bill flat at the check level.
async function scrapeSearchMonitorPage(params: {
  teamId: string;
  teamFlags: TeamFlags | null;
  monitorId: string;
  checkId: string;
  url: string;
  judgePrompt: string;
}): Promise<ScrapeSearchResult | null> {
  const scrapeId = uuidv7();
  const scrapeOptions = scrapeRequestSchema.parse({
    url: params.url,
    formats: [
      { type: "markdown" },
      { type: "json", schema: verdictJsonSchema, prompt: params.judgePrompt },
    ],
    timeout: 20000,
    origin: "monitor",
  });

  await logRequest({
    id: scrapeId,
    kind: "scrape",
    api_version: "v2",
    team_id: params.teamId,
    origin: "monitor",
    integration: null,
    target_hint: params.url,
    zeroDataRetention: false,
    api_key_id: null,
  });

  const job: NuQJob<ScrapeJobData> = {
    id: scrapeId,
    status: "active",
    createdAt: new Date(),
    priority: 20,
    data: {
      mode: "single_urls",
      url: params.url,
      team_id: params.teamId,
      scrapeOptions,
      internalOptions: {
        teamId: params.teamId,
        // Monitors do no in-pipeline blocklist enforcement (business rule);
        // search results are filtered via isBlocked before scraping.
        orgId: null,
        saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
        bypassBilling: true,
        zeroDataRetention: false,
        // Safe Mode resolves per-URL at the scrapeURL backstop from these flags.
        teamFlags: params.teamFlags ?? undefined,
        // The JSON format is the search judge; trace it as part of the check.
        llmTelemetry: {
          functionId: "monitor/searchJudge",
          metadata: monitorTelemetryMetadata("monitor_search_judge", {
            teamId: params.teamId,
            monitorId: params.monitorId,
            monitorCheckId: params.checkId,
          }),
        },
      },
      skipNuq: true,
      origin: "monitor",
      integration: null,
      billing: { endpoint: "monitor", jobId: params.checkId },
      zeroDataRetention: false,
      apiKeyId: null,
    },
  };

  const doc = await processJobInternal(job);
  if (!doc) return null;
  return {
    json: doc.json ?? null,
    markdown: doc.markdown ?? "",
    metadata: {
      publishedTime: doc.metadata?.publishedTime ?? null,
      modifiedTime: doc.metadata?.modifiedTime ?? null,
    },
  };
}

function summarize(pages: PageResult[]) {
  return {
    totalPages: pages.length,
    same: pages.filter(page => page.status === "same").length,
    changed: pages.filter(page => page.status === "changed").length,
    new: pages.filter(page => page.status === "new").length,
    removed: pages.filter(page => page.status === "removed").length,
    error: pages.filter(page => page.status === "error").length,
  };
}

/// `false` means the settle did not land: the hold is still out there and this
/// run is not billed. Only the caller can decide what to record for that, so it
/// is returned rather than swallowed.
async function billMonitorCheck(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  actualCredits: number;
  lockId: string | null;
  /** The team's org. Null means none could be named: the settle still goes
   * through firebill when it is configured, but is unreportable to a partner. */
  orgId: string | null;
}): Promise<boolean> {
  let settled = true;
  if (params.lockId) {
    settled = await autumnService.finalizeCreditsLock({
      lockId: params.lockId,
      action: "confirm",
      overrideValue: params.actualCredits,
      properties: {
        source: "monitorCheck",
        endpoint: "monitor",
        jobId: params.check.id,
      },
      team: params.orgId
        ? { teamId: params.monitor.team_id, orgId: params.orgId }
        : undefined,
      // Confirm only: a release bills nothing, so there is nothing to report.
      externalRequestId: params.check.partner_run_token,
      // What the lock reserved. firebill adds it back to the ghost's reported
      // balance, which Autumn returns net of this very hold.
      heldValue: params.check.reserved_credits,
    });
  }

  if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION)
    return settled;

  // The settle did not land, so Autumn has nothing and the hold expires by
  // itself. Debiting the team's ledger anyway charges them for a run Autumn
  // never billed, leaving the two ledgers disagreeing while the row already
  // says `failed`. `autumnTrackInRequest` below would be a lie too: it tells
  // the batch Autumn already has this, and it does not.
  //
  // So nothing is charged. Firecrawl absorbs the run — the direction this path
  // errs everywhere else — and `billing_status: "failed"` plus the error beside
  // it are how it is found.
  if (!settled) {
    logger.error("Not billing a monitor check whose settle did not land", {
      monitorId: params.monitor.id,
      checkId: params.check.id,
      lockId: params.lockId,
      actualCredits: params.actualCredits,
    });
    return false;
  }

  await getBillingQueue().add(
    "bill_team",
    {
      team_id: params.monitor.team_id,
      org_id: params.orgId,
      credits: params.actualCredits,
      billing: { endpoint: "monitor", jobId: params.check.id },
      is_extract: false,
      timestamp: new Date().toISOString(),
      originating_job_id: params.check.id,
      api_key_id: null,
      autumnTrackInRequest: Boolean(params.lockId),
    },
    {
      // Deterministic per check so a re-finalize (e.g. the reconciler re-running this
      // check after the finalize lock TTL expired mid-finalize) re-enqueues the SAME
      // job id and the billing queue dedups it instead of charging the team twice.
      jobId: `monitor-bill-${params.check.id}`,
      priority: 10,
    },
  );

  return settled;
}

async function sendNotifications(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  pages: PageResult[];
}): Promise<{
  webhook?: unknown;
  email?: unknown;
  slack?: unknown;
  inApp?: unknown;
}> {
  const payload = {
    monitorId: params.monitor.id,
    checkId: params.check.id,
    status: params.check.status,
    summary: toSummaryObject(params.check),
  };

  const nonSamePages = params.pages.filter(page => page.status !== "same");

  // Never throws, and goes first so a failing channel below cannot skip it.
  const inAppStatus = await recordMonitorInAppNotification({
    monitor: params.monitor,
    check: params.check,
    pages: nonSamePages.map(page => ({
      url: page.url,
      status: page.status,
      judgment: page.judgment ?? null,
    })),
  });

  let webhookStatus: unknown = { attempted: false };
  if (params.monitor.webhook) {
    const sender = await createWebhookSender({
      teamId: params.monitor.team_id,
      jobId: params.check.id,
      webhook: params.monitor.webhook as any,
      v0: false,
    });
    try {
      const result = await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {
        success: params.check.status === "completed",
        data: [payload],
        error: params.check.error ?? undefined,
        awaitWebhook: true,
      });
      webhookStatus = {
        attempted: result?.attempted ?? false,
        success: result?.delivered === true,
        delivered: result?.delivered === true,
        queued: result?.queued === true,
        skipped: result?.skipped === true,
      };
    } catch (error) {
      webhookStatus = {
        attempted: true,
        success: false,
        delivered: false,
        error: error instanceof Error ? error.message : String(error),
      };
    }
  }

  // Pull diff text for up to 5 meaningful changed pages so the email leads with
  // the diff. Errors swallowed per-page so one GCS hiccup doesn't drop the alert.
  const diffEligible = nonSamePages
    .filter(
      p => p.status === "changed" && (!p.judgment || p.judgment.meaningful),
    )
    .slice(0, 5);
  const diffTextByUrl = new Map<string, string>();
  await Promise.all(
    diffEligible.map(async page => {
      if (!page.diff_gcs_key) return;
      try {
        const artifact = await getMonitorDiffArtifact(page.diff_gcs_key);
        const text =
          artifact?.kind === "markdown"
            ? artifact.text
            : artifact?.markdown?.text;
        if (text) diffTextByUrl.set(page.url, text);
      } catch (error) {
        logger.warn("Failed to load diff artifact for email", {
          error,
          url: page.url,
        });
      }
    }),
  );

  const emailStatus = await sendMonitoringEmailSummary({
    monitor: params.monitor,
    check: params.check,
    pages: nonSamePages.map(page => ({
      url: page.url,
      status: page.status,
      error: page.error,
      judgment: page.judgment ?? null,
      diffText: diffTextByUrl.get(page.url) ?? null,
    })),
  });

  let slackStatus: unknown = { attempted: false };
  try {
    slackStatus = await sendMonitoringSlackSummary({
      monitor: params.monitor,
      check: params.check,
      pages: nonSamePages.map(page => ({
        url: page.url,
        status: page.status,
        judgment: page.judgment ?? null,
      })),
    });
  } catch (error) {
    logger.warn("Slack monitor summary threw", {
      error,
      monitorId: params.monitor.id,
      checkId: params.check.id,
    });
    slackStatus = {
      attempted: true,
      success: false,
      error: error instanceof Error ? error.message : String(error),
    };
  }

  return {
    webhook: webhookStatus,
    email: emailStatus,
    slack: slackStatus,
    inApp: inAppStatus,
  };
}

async function enqueueMonitorScrapeTarget(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  target: MonitorTarget;
  targetRun: Extract<MonitorTargetRun, { type: "scrape" }>;
}): Promise<Extract<MonitorTargetRun, { type: "scrape" }>> {
  if (params.target.type !== "scrape") {
    throw new Error("Expected scrape target");
  }

  // Safe Mode applies to monitor scrapes: resolve per-URL at the backstop from
  // the team's flags. orgId stays null (monitors' separate no-blocklist rule).
  const acuc = await getACUCTeam(params.monitor.team_id);

  for (const [index, url] of params.target.urls.entries()) {
    const scrapeId = params.targetRun.expectedJobs[index];
    const scrapeOptions = scrapeRequestSchema.parse({
      url,
      ...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
      origin: "monitor",
    });

    await logRequest({
      id: scrapeId,
      kind: "scrape",
      api_version: "v2",
      team_id: params.monitor.team_id,
      origin: "monitor",
      integration: null,
      target_hint: url,
      zeroDataRetention: false,
      api_key_id: null,
    });

    await addScrapeJob(
      {
        mode: "single_urls",
        url,
        team_id: params.monitor.team_id,
        scrapeOptions,
        internalOptions: {
          teamId: params.monitor.team_id,
          // Monitors do no in-pipeline blocklist enforcement (business rule).
          orgId: null,
          saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
          bypassBilling: true,
          zeroDataRetention: false,
          teamFlags: acuc?.flags ?? undefined,
        },
        origin: "monitor",
        integration: null,
        billing: { endpoint: "monitor", jobId: params.check.id },
        zeroDataRetention: false,
        apiKeyId: null,
        monitoring: {
          monitorId: params.monitor.id,
          checkId: params.check.id,
          targetId: params.target.id,
          source: "explicit",
        },
      },
      scrapeId,
      20,
    );
  }

  return params.targetRun;
}

async function enqueueMonitorCrawlTarget(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  target: MonitorTarget;
  targetRun: Extract<MonitorTargetRun, { type: "crawl" }>;
}): Promise<Extract<MonitorTargetRun, { type: "crawl" }>> {
  if (params.target.type !== "crawl") {
    throw new Error("Expected crawl target");
  }

  const crawlId = params.targetRun.crawlId;
  const body = crawlRequestSchema.parse({
    url: params.target.url,
    ...(params.target.crawlOptions ?? {}),
    scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
    origin: "monitor",
  }) as CrawlRequest;

  await logRequest({
    id: crawlId,
    kind: "crawl",
    api_version: "v2",
    team_id: params.monitor.team_id,
    origin: "monitor",
    integration: null,
    target_hint: body.url,
    zeroDataRetention: false,
    api_key_id: null,
    creditsShards: requestCreditsShards(
      body.limit ?? MONITOR_CHECK_PAGE_SCAN_LIMIT,
    ),
  });

  const crawlerOptions = {
    ...body,
    url: undefined,
    scrapeOptions: undefined,
    prompt: undefined,
  };

  // Safe Mode applies to monitor crawls: each child resolves it per-URL at the
  // backstop from the team's flags. orgId stays null (monitors' no-blocklist rule).
  const acuc = await getACUCTeam(params.monitor.team_id);

  const sc: StoredCrawl = {
    originUrl: body.url,
    crawlerOptions: toV0CrawlerOptions(crawlerOptions),
    scrapeOptions: body.scrapeOptions,
    internalOptions: {
      disableSmartWaitCache: true,
      teamId: params.monitor.team_id,
      // Monitors do no in-pipeline blocklist enforcement (business rule).
      orgId: null,
      saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
      zeroDataRetention: false,
      bypassBilling: true,
      teamFlags: acuc?.flags ?? undefined,
    },
    team_id: params.monitor.team_id,
    createdAt: Date.now(),
    maxConcurrency: body.maxConcurrency,
    zeroDataRetention: false,
  };

  const crawler = crawlToCrawler(crawlId, sc, null);
  try {
    sc.robots = await crawler.getRobotsTxt(
      body.scrapeOptions.skipTlsVerification,
    );
  } catch {
    // Non-fatal robots fetch failure, same as the public crawl controller.
  }

  sc.queueBackend = await resolveNewGroupBackend(sc.team_id);
  await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000, {
    backend: sc.queueBackend,
    maxConcurrency: sc.maxConcurrency,
    delaySeconds: sc.crawlerOptions?.delay,
  });
  await saveCrawl(crawlId, sc);
  await markCrawlActive(crawlId);

  await _addScrapeJobToBullMQ(
    {
      url: body.url,
      mode: "kickoff",
      team_id: params.monitor.team_id,
      crawlerOptions,
      scrapeOptions: sc.scrapeOptions,
      internalOptions: sc.internalOptions,
      origin: "monitor",
      integration: null,
      billing: { endpoint: "monitor", jobId: params.check.id },
      crawl_id: crawlId,
      v1: true,
      zeroDataRetention: false,
      apiKeyId: null,
      monitoring: {
        monitorId: params.monitor.id,
        checkId: params.check.id,
        targetId: params.target.id,
        source: "discovered",
      },
    },
    uuidv7(),
  );

  return params.targetRun;
}

// Runs inline, persisting onto the same monitor_pages / monitor_check_pages
// tables the reconciler tallies.
// Bound the inline finalize writes: under write-pool exhaustion they can wait forever,
// stranding the check until the 10-min reaper. Throw so the catch fails it fast instead.
const MONITOR_FINALIZE_WRITE_TIMEOUT_MS = 60_000;

class MonitorFinalizeTimeoutError extends Error {
  constructor(what: string, ms: number) {
    super(`${what} exceeded ${ms}ms`);
    this.name = "MonitorFinalizeTimeoutError";
  }
}

// Reject (not resolve) on timeout so a stalled write fails fast into the catch path.
// The race rejects but can't truly cancel work(); we abort a signal so work() can
// cooperatively stop issuing further writes, otherwise a stalled write may land
// AFTER the catch has marked the check terminal — corrupting cross-run dedup state.
export async function withFinalizeTimeout<T>(
  work: (signal: AbortSignal) => Promise<T>,
  what: string,
  ms: number = MONITOR_FINALIZE_WRITE_TIMEOUT_MS,
): Promise<T> {
  const controller = new AbortController();
  let timer: ReturnType<typeof setTimeout> | undefined;
  const timeout = new Promise<never>((_, reject) => {
    timer = setTimeout(() => {
      controller.abort();
      reject(new MonitorFinalizeTimeoutError(what, ms));
    }, ms);
  });
  try {
    return await Promise.race([work(controller.signal), timeout]);
  } finally {
    if (timer) clearTimeout(timer);
  }
}

async function runMonitorSearchTarget(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  target: MonitorTarget;
}): Promise<{
  pages: PageResult[];
  resultCount: number;
  matches: number;
  summary: string;
  judgeDegraded: boolean;
  degradedReason: string | null;
  // Flat credits, recorded onto target_results by the caller.
  searchCredits: number;
  judgeCredits: number;
  resultsJudged: number;
}> {
  if (params.target.type !== "search") {
    return {
      pages: [],
      resultCount: 0,
      matches: 0,
      summary: "",
      judgeDegraded: false,
      degradedReason: null,
      searchCredits: 0,
      judgeCredits: 0,
      resultsJudged: 0,
    };
  }
  const { monitor, check, target } = params;
  const goalVersion = computeGoalVersion(
    monitor.goal,
    monitor.name,
    target.queries,
  );

  // Rebuild per-URL dedup memory + event index from this target's prior pages.
  const priorPages = await listActiveMonitorPages({
    monitorId: monitor.id,
    targetId: target.id,
  });
  const { knownPages, knownEvents } = reconstructKnownState(
    priorPages,
    goalVersion,
  );

  // Same blocklist gate prod scrapes use, applied per team (honors unblockedDomains).
  const acuc = await getACUCTeam(monitor.team_id);
  const teamFlags = acuc?.flags ?? null;

  const result = await runSearchTarget({
    monitor: {
      id: monitor.id,
      teamId: monitor.team_id,
      goal: monitor.goal,
      subject: monitor.name,
      // Read fresh each check so a PATCH/UI toggle takes effect next check.
      judgeEnabled: Boolean(monitor.judge_enabled),
    },
    target: {
      id: target.id,
      queries: target.queries,
      searchWindow: target.searchWindow,
      // depth/alertMode aren't API-settable but stored targets may carry them (back-compat); pass through.
      alertMode: target.alertMode ?? "first_match",
      includeDomains: target.includeDomains,
      excludeDomains: target.excludeDomains,
      recheckAfter: target.recheckAfter,
      maxResults: target.maxResults,
      depth: target.depth,
    },
    monitorCheckId: check.id,
    scrapePage: ({ url, judgePrompt }) =>
      scrapeSearchMonitorPage({
        teamId: monitor.team_id,
        teamFlags,
        monitorId: monitor.id,
        checkId: check.id,
        url,
        judgePrompt,
      }),
    isBlocked: url =>
      isUrlBlocked(url, teamFlags, {
        team_id: monitor.team_id,
        org_id: acuc?.org_id ?? null,
        origin: "monitor.search",
      }),
    goalVersion,
    knownPages,
    knownEvents,
    zeroDataRetention: false,
    usOnly: isUsOnly(teamFlags),
    logger: logger.child({
      monitorId: monitor.id,
      checkId: check.id,
      targetId: target.id,
    }),
  });

  const searchCredits = result.searchCredits;
  const judgeCredits = result.judgeCredits;

  // Search pages carry no per-page credit — billed once at check level.
  const pages: PageResult[] = result.pageUpserts.map(upsert => {
    const status = searchStatusToPageStatus(upsert.status);
    return {
      check_id: check.id,
      monitor_id: monitor.id,
      team_id: monitor.team_id,
      target_id: target.id,
      url: upsert.url,
      url_hash: upsert.urlHash,
      status,
      metadata: upsert.metadata,
      judgment: upsert.judgment ?? null,
      emailStatus: status,
    };
  });

  await withFinalizeTimeout(async signal => {
    // Per-check rows first: idempotent (delete+insert replace) and not read across runs.
    // Clear rows from a prior (crashed/redelivered) run so the insert is a replace, not a duplicate.
    if (signal.aborted) return;
    await deleteMonitorCheckPages({ checkId: check.id, targetId: target.id });
    if (signal.aborted) return;
    await insertMonitorCheckPages(pages);

    // Durable cross-run dedup baseline last, so a timeout before this point leaves
    // monitor_pages untouched and the next run re-alerts. One bulk upsert (~3 round-
    // trips) instead of ~2N sequential pool acquisitions that stalled finalization;
    // the signal lets an aborted finalize skip the write entirely.
    await bulkUpsertMonitorPages({
      monitorId: monitor.id,
      teamId: monitor.team_id,
      targetId: target.id,
      checkId: check.id,
      rows: pages.map(page => ({
        url: page.url,
        urlHash: page.url_hash,
        status: page.status,
        metadata: page.metadata as Record<string, unknown>,
        source: "discovered",
        scrapeId: null,
      })),
      abortSignal: signal,
    });
  }, "monitor search page-write tail");

  for (const page of pages) {
    if (page.status !== "new" && page.status !== "error") continue;
    await sendMonitorPageWebhook({
      teamId: monitor.team_id,
      monitorId: monitor.id,
      checkId: check.id,
      url: page.url,
      status: page.status,
      error: page.error ?? null,
      judgment: page.judgment ?? null,
    });
  }

  return {
    pages,
    resultCount: result.resultCount,
    matches: result.matches,
    summary: result.summary,
    judgeDegraded: result.judgeDegraded,
    degradedReason: result.degradedReason,
    searchCredits,
    judgeCredits,
    resultsJudged: result.resultsJudged,
  };
}

// Find a completed search target run so a redelivered check can restore its
// figures instead of re-running (and re-billing) it.
export function findCompletedSearchTargetRun(
  targetResults: unknown,
  targetId: string,
): Record<string, unknown> | null {
  if (!Array.isArray(targetResults)) return null;
  const match = targetResults.find(
    tr =>
      tr != null &&
      typeof tr === "object" &&
      (tr as { type?: unknown }).type === "search" &&
      (tr as { targetId?: unknown }).targetId === targetId &&
      (tr as { searchCompleted?: unknown }).searchCompleted === true,
  );
  return (match as Record<string, unknown>) ?? null;
}

async function releaseUnpersistedMonitorHold(
  monitor: MonitorRow,
  checkId: string,
  lockId: string | null,
  orgId: string | null,
): Promise<void> {
  if (!lockId) return;
  const latest = await getMonitorCheckForUpdate(
    monitor.team_id,
    monitor.id,
    checkId,
  );
  // A duplicate delivery can receive the same lock ID as the finalizer.
  // Only release an unpersisted hold after no running writer can adopt it.
  if (
    !latest ||
    (latest.status !== "queued" &&
      latest.status !== "running" &&
      latest.autumn_lock_id !== lockId)
  ) {
    await autumnService.finalizeCreditsLock({
      lockId,
      action: "release",
      properties: {
        source: "monitorCheck",
        endpoint: "monitor",
        jobId: checkId,
      },
      team: orgId ? { teamId: monitor.team_id, orgId } : undefined,
    });
  }
}

export async function processMonitorCheckJob(
  job: MonitorCheckJobData,
): Promise<void> {
  const monitor = await getMonitorForUpdate(job.teamId, job.monitorId);
  if (!monitor) {
    throw new Error("Monitor not found");
  }

  const initialCheck = await getMonitorCheckForUpdate(
    job.teamId,
    job.monitorId,
    job.checkId,
  );
  if (!initialCheck) {
    throw new Error("Monitor check not found");
  }
  if (initialCheck.status !== "queued" && initialCheck.status !== "running") {
    return;
  }

  const started =
    initialCheck.status === "running"
      ? initialCheck
      : await updateMonitorCheckIfStatus(job.checkId, "queued", {
          status: "running",
          started_at: new Date().toISOString(),
        });
  if (!started) return;
  let check: MonitorCheckRow = started;

  // One org lookup for the whole check job — the billing service no longer
  // makes it, so every hold, settle and release below shares this one. A
  // failure answers null, which is what the lookup inside the biller did.
  const orgId = await orgIdForTeam(monitor.team_id);
  const admissionCheck = await getMonitorCheckForUpdate(
    job.teamId,
    job.monitorId,
    job.checkId,
  );
  if (!admissionCheck || admissionCheck.status !== "running") return;
  check = admissionCheck;
  const partnerJobToken = check.partner_run_token
    ? null
    : monitor.partner_job_token;

  let lockId: string | null = null;
  try {
    const lock = orgId
      ? await autumnService.lockCredits({
          teamId: monitor.team_id,
          orgId,
          value: check.estimated_credits ?? 1,
          lockId: `monitor_${check.id}`,
          expiresAt: Date.now() + 60 * 60 * 1000,
          properties: {
            source: "monitorCheck",
            endpoint: "monitor",
            jobId: check.id,
          },
          // Arms firebill's partner gate; NULL is today's lock. Withheld once this
          // check holds a run token: a redelivery is the same occurrence, and
          // asking twice would orphan the first token.
          partnerJobToken,
        })
      : // No org, no customer to hold against — the same answers the hold gave
        // when it could not name one: proceed unlocked, except for a gated run,
        // which is unauthorized until a partner says otherwise.
        partnerJobToken
        ? ({ status: "denied", reason: "gate_unavailable" } as const)
        : ({ status: "skipped" } as const);

    if (lock.status === "denied") {
      const revoked = lock.reason === "job_revoked";
      const skipped = await updateMonitorCheckIfRunning(check.id, {
        status: "skipped_no_credits",
        finished_at: new Date().toISOString(),
        actual_credits: 0,
        billing_status: "not_applicable",
        error: revoked
          ? MONITOR_CHECK_REVOKED_ERROR
          : MONITOR_CHECK_NO_CREDITS_ERROR,
      });

      if (!skipped) return;
      check = skipped;

      // A revoked job never becomes unrevoked; see MONITOR_GATE_REVOKED_STREAK
      // for why it is waited out rather than acted on at once.
      const paused =
        revoked &&
        (await countRecentConsecutiveSkippedForCredits({
          teamId: monitor.team_id,
          monitorId: monitor.id,
          limit: MONITOR_GATE_REVOKED_STREAK,
        })) >= MONITOR_GATE_REVOKED_STREAK;

      if (paused) {
        await pauseMonitor(monitor.id);
        logger.warn("Paused monitor: the partner has revoked this job", {
          monitorId: monitor.id,
          checkId: check.id,
          teamId: monitor.team_id,
          consecutiveSkips: MONITOR_GATE_REVOKED_STREAK,
        });
      }

      // Reads status off the object it is given, not the row — a paused
      // monitor must not be handed a next_run_at.
      await updateMonitorScheduleAfterRun({
        monitor: paused ? { ...monitor, status: "paused" } : monitor,
        check,
      });

      logger.info("Skipped monitor check: no credit authority allowed it", {
        monitorId: monitor.id,
        checkId: check.id,
        teamId: monitor.team_id,
        reason: lock.reason,
      });
      return;
    }

    lockId = lock.status === "locked" ? lock.lockId : null;

    const reserved = await updateMonitorCheckIfRunning(check.id, {
      autumn_lock_id: lockId,
      // A token already on the row wins: the gate was not re-asked, so there
      // is no newer one, and null would lose the authorized operation.
      partner_run_token:
        check.partner_run_token ??
        (lock.status === "locked" ? (lock.operationToken ?? null) : null),
      reserved_credits: lockId ? (check.estimated_credits ?? 1) : null,
      billing_status: lockId ? "reserved" : "not_applicable",
    });

    if (!reserved) {
      await releaseUnpersistedMonitorHold(monitor, check.id, lockId, orgId);
      return;
    }
    check = reserved;

    await markMonitorRunning({
      monitorId: monitor.id,
      checkId: job.checkId,
    });

    trackMonitorCheckStartedInterest({ monitor, check }).catch(error =>
      logger.warn("Failed to track monitor target interest", {
        error,
        monitorId: monitor.id,
        checkId: check.id,
        eventType: "check_started",
      }),
    );

    const targetResults = monitor.targets.map(createMonitorTargetRun);
    const initialized = await updateMonitorCheckIfRunning(check.id, {
      target_results: targetResults,
    });
    if (!initialized) return;

    for (const [index, target] of monitor.targets.entries()) {
      const targetRun = targetResults[index];
      if (target.type === "scrape" && targetRun.type === "scrape") {
        await enqueueMonitorScrapeTarget({ monitor, check, target, targetRun });
      } else if (target.type === "crawl" && targetRun.type === "crawl") {
        await enqueueMonitorCrawlTarget({ monitor, check, target, targetRun });
      } else if (target.type === "search" && targetRun.type === "search") {
        // Redelivery after inline work finished but before ack: restore persisted
        // figures instead of re-running, which would re-bill and re-scrape.
        const priorRun = findCompletedSearchTargetRun(
          initialCheck.target_results,
          target.id,
        );
        if (priorRun) {
          Object.assign(targetRun, priorRun);
          targetRun.searchCompleted = true;
          continue;
        }
        // Search runs synchronously; fold its outcome into target_results.
        const searchResult = await runMonitorSearchTarget({
          monitor,
          check,
          target,
        });
        targetRun.resultCount = searchResult.resultCount;
        targetRun.matches = searchResult.matches;
        targetRun.summary = searchResult.summary;
        targetRun.judgeDegraded = searchResult.judgeDegraded;
        targetRun.degradedReason = searchResult.degradedReason;
        targetRun.searchCredits = searchResult.searchCredits;
        targetRun.judgeCredits = searchResult.judgeCredits;
        targetRun.resultsJudged = searchResult.resultsJudged;
        // Set last, after credits are stamped, so the reconciler never finalizes with credits at 0.
        targetRun.searchCompleted = true;
        // Persist searchCompleted now so a crash/redelivery short-circuits via
        // findCompletedSearchTargetRun instead of re-running and re-billing.
        const persisted = await withFinalizeTimeout(
          signal =>
            signal.aborted
              ? Promise.resolve(null)
              : // Atomic guard: if this write outran the timeout and the catch
                // already failed the check, no-op instead of stamping searchCompleted.
                updateMonitorCheckIfRunning(check.id, {
                  target_results: targetResults,
                }),
          "monitor search searchCompleted flush",
        );
        if (!persisted) return;
      }
    }

    const stored = await updateMonitorCheckIfRunning(check.id, {
      target_results: targetResults,
    });

    // Search targets finish inline, so a search-only check is complete here.
    // Any failure or lost lease leaves it to the reconciler, as before.
    if (stored && monitor.targets.every(target => target.type === "search")) {
      await reconcileMonitorCheckCandidate(stored, () => {}).catch(error =>
        logger.warn("Failed to finalize monitor check inline", {
          error,
          monitorId: monitor.id,
          checkId: check.id,
        }),
      );
    }
  } catch (error) {
    // Atomically flip running -> failed. Returns null when the check already
    // reached a terminal status — i.e. the reconciler finalized it (completed,
    // billed, lock confirmed) before this late catch ran. In that case we must
    // not clobber its terminal state or release its now-confirmed credit lock;
    // the reconciler already owns billing, notifications, and scheduling.
    const failed = await updateMonitorCheckIfRunning(check.id, {
      status: "failed",
      finished_at: new Date().toISOString(),
      error: error instanceof Error ? error.message : String(error),
    });

    if (!failed) {
      await releaseUnpersistedMonitorHold(monitor, check.id, lockId, orgId);
      throw error;
    }
    check = failed;

    // The claim owns the persisted hold; a failed reservation write can also
    // leave this handler with a newly acquired hold that was never stored.
    let released = true;
    for (const ownedLockId of new Set([failed.autumn_lock_id, lockId])) {
      if (!ownedLockId) continue;
      const settled = await autumnService
        .finalizeCreditsLock({
          lockId: ownedLockId,
          action: "release",
          properties: {
            source: "monitorCheck",
            endpoint: "monitor",
            jobId: check.id,
          },
          team: orgId ? { teamId: monitor.team_id, orgId } : undefined,
        })
        .catch(releaseError => {
          logger.warn("Failed to release monitor check credit lock", {
            error: releaseError,
            monitorId: monitor.id,
            checkId: check.id,
            lockId: ownedLockId,
          });
          return false;
        });
      released = released && settled;
    }
    check = await updateMonitorCheck(failed.id, {
      billing_status:
        (failed.autumn_lock_id || lockId) && released ? "released" : "failed",
    });

    if (
      await claimMonitorNotification(check.id).catch(error => {
        logger.warn(
          "Failed to claim monitor notification; continuing without dedupe",
          {
            error,
            monitorId: monitor.id,
            checkId: check.id,
          },
        );
        return true;
      })
    ) {
      const notificationStatus = await sendNotifications({
        monitor,
        check,
        pages: [],
      }).catch(err => {
        logger.warn("Failed to send monitor failure notifications", {
          error: err,
        });
        return null;
      });
      if (notificationStatus) {
        check = await updateMonitorCheck(check.id, {
          notification_status: notificationStatus,
        }).catch(updateError => {
          logger.warn("Failed to record monitor failure notification status", {
            error: updateError,
            monitorId: monitor.id,
            checkId: check.id,
          });
          return check;
        });
      }
    }

    await updateMonitorScheduleAfterRun({
      monitor,
      check,
    });

    throw error;
  }
}

async function processRemovedPagesForCompletedCrawls(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  targetResults: any[];
}): Promise<void> {
  for (const target of params.targetResults) {
    if (target?.type !== "crawl" || target.removedProcessed) continue;

    const group = await crawlGroup.getGroup(target.crawlId);
    if (group?.status !== "completed") continue;

    const checkPages = await listMonitorCheckPages({
      teamId: params.monitor.team_id,
      monitorId: params.monitor.id,
      checkId: params.check.id,
      limit: MONITOR_CHECK_PAGE_SCAN_LIMIT,
      skip: 0,
    });
    const seen = new Set(
      checkPages
        .filter(page => page.target_id === target.targetId)
        .map(page => page.url_hash.toString("hex")),
    );
    const activePages = await listActiveMonitorPages({
      monitorId: params.monitor.id,
      targetId: target.targetId,
    });

    const removed: MonitorCheckPageInsert[] = [];
    for (const previous of activePages) {
      if (seen.has(previous.url_hash.toString("hex"))) continue;
      await upsertMonitorPage({
        monitorId: params.monitor.id,
        teamId: params.monitor.team_id,
        targetId: target.targetId,
        url: previous.url,
        source: previous.source,
        checkId: params.check.id,
        scrapeId: previous.last_scrape_id,
        status: "removed",
        metadata: previous.metadata,
      });
      removed.push({
        check_id: params.check.id,
        monitor_id: params.monitor.id,
        team_id: params.monitor.team_id,
        target_id: target.targetId,
        url: previous.url,
        url_hash: previous.url_hash,
        status: "removed" as const,
        previous_scrape_id: previous.last_scrape_id,
        current_scrape_id: null,
      });
    }

    await insertMonitorCheckPages(removed);
    target.removedProcessed = true;
  }
}

// The first target still in flight, with the figures that decided it, or null
// once every target is done.
async function findUnfinishedMonitorTarget(
  check: MonitorCheckRow,
  monitor?: MonitorRow,
): Promise<Record<string, unknown> | null> {
  let targetResults = Array.isArray(check.target_results)
    ? (check.target_results as any[])
    : [];

  if (targetResults.length === 0) {
    if (!monitor) return { reason: "no_target_results" };

    targetResults = await recoverTargetRunsFromRecordedPages({
      monitor,
      check,
    });
    if (targetResults.length === 0) return { reason: "no_target_results" };
  }

  for (const target of targetResults) {
    if (target?.type === "search") {
      // Not complete until the inline search has stamped its credits.
      if (!target.searchCompleted) {
        return {
          type: "search",
          targetId: target.targetId,
          searchCompleted: false,
        };
      }
    } else if (target?.type === "scrape") {
      const expected = Array.isArray(target.expectedJobs)
        ? target.expectedJobs.length
        : 0;
      const recorded = await countMonitorCheckPages({
        checkId: check.id,
        targetId: target.targetId,
      });
      if (recorded < expected) {
        return {
          type: "scrape",
          targetId: target.targetId,
          recorded,
          expected,
        };
      }
    } else if (target?.type === "crawl") {
      const group = await crawlGroup.getGroup(target.crawlId);
      if (!group) {
        return {
          type: "crawl",
          targetId: target.targetId,
          crawlId: target.crawlId,
          groupStatus: null,
        };
      }

      const stats = await scrapeQueue.getGroupNumericStats(
        target.crawlId,
        logger,
      );
      const active = stats.active ?? 0;
      const queued = stats.queued ?? 0;
      const backlog = stats.backlog ?? 0;
      if (group.status === "active" || active + queued + backlog > 0) {
        return {
          type: "crawl",
          targetId: target.targetId,
          crawlId: target.crawlId,
          groupStatus: group.status,
          active,
          queued,
          backlog,
        };
      }
    }
  }

  return null;
}

async function failStaleMonitorCheck(params: {
  monitor: MonitorRow;
  check: MonitorCheckRow;
  /** See billMonitorCheck's orgId. */
  orgId: string | null;
}): Promise<boolean> {
  if (!isMonitorCheckStale(params.check, new Date(), params.monitor.targets))
    return false;

  const error = MONITOR_CHECK_STALE_ERROR;
  const claimed = await updateMonitorCheckIfRunning(params.check.id, {
    status: "failed",
    finished_at: new Date().toISOString(),
    actual_credits: 0,
    error,
  });
  if (!claimed) return true;

  let released = true;
  if (claimed.autumn_lock_id) {
    released = await autumnService
      .finalizeCreditsLock({
        lockId: claimed.autumn_lock_id,
        action: "release",
        properties: {
          source: "monitorCheck",
          endpoint: "monitor",
          jobId: params.check.id,
        },
        team: params.orgId
          ? { teamId: params.monitor.team_id, orgId: params.orgId }
          : undefined,
      })
      .catch(releaseError => {
        logger.warn("Failed to release stale monitor check credit lock", {
          error: releaseError,
          monitorId: params.monitor.id,
          checkId: params.check.id,
          lockId: claimed.autumn_lock_id,
        });
        return false;
      });
  }

  const finalized = await updateMonitorCheck(claimed.id, {
    billing_status: !claimed.autumn_lock_id
      ? "not_applicable"
      : released
        ? "released"
        : "failed",
  });
  let withNotifications = finalized;
  if (await claimMonitorNotification(params.check.id)) {
    const notificationStatus = await sendNotifications({
      monitor: params.monitor,
      check: finalized,
      pages: [],
    }).catch(notificationError => {
      logger.warn("Failed to send stale monitor check notifications", {
        error: notificationError,
        monitorId: params.monitor.id,
        checkId: params.check.id,
      });
      return {
        webhook: {
          attempted: !!params.monitor.webhook,
          success: false,
          error:
            notificationError instanceof Error
              ? notificationError.message
              : String(notificationError),
        },
        email: {
          attempted: !!params.monitor.notification?.email?.enabled,
          success: false,
          error:
            notificationError instanceof Error
              ? notificationError.message
              : String(notificationError),
        },
      };
    });

    withNotifications = await updateMonitorCheck(params.check.id, {
      notification_status: notificationStatus,
    }).catch(updateError => {
      logger.warn("Failed to record stale monitor check notification status", {
        error: updateError,
        monitorId: params.monitor.id,
        checkId: params.check.id,
      });
      return finalized;
    });
  }

  if (params.monitor.current_check_id === params.check.id) {
    await updateMonitorScheduleAfterRun({
      monitor: params.monitor,
      check: withNotifications,
      summary: toSummaryObject(withNotifications),
    });
  }

  logger.warn("Failed stale monitor check", {
    monitorId: params.monitor.id,
    checkId: params.check.id,
    startedAt: params.check.started_at,
    timeoutMs: monitorCheckStaleTimeoutMs(params.check, params.monitor.targets),
  });

  return true;
}

let runningCheckScan: Parameters<typeof listRunningMonitorChecks>[1];
let reconcilingRunningChecks = false;

export async function reconcileRunningMonitorChecks(
  limit: number = 50,
): Promise<void> {
  if (!Number.isInteger(limit) || limit < 1 || limit > 50) {
    throw new RangeError(
      "Monitor reconciliation limit must be between 1 and 50.",
    );
  }
  if (reconcilingRunningChecks) return;
  reconcilingRunningChecks = true;
  try {
    await reconcileRunningMonitorCheckPage(limit);
  } finally {
    reconcilingRunningChecks = false;
  }
}

async function reconcileRunningMonitorCheckPage(limit: number): Promise<void> {
  const [oldestResult, newestResult] = await Promise.allSettled([
    listRunningMonitorChecks(limit),
    listRunningMonitorChecks(limit, { newest: true }),
  ]);
  if (oldestResult.status === "rejected") throw oldestResult.reason;
  if (newestResult.status === "rejected") throw newestResult.reason;
  const oldest = oldestResult.value;
  const newest = newestResult.value;
  runningCheckScan ??= newest[0]
    ? { through: { created_at: newest[0].created_at, id: newest[0].id } }
    : undefined;
  const scan = runningCheckScan;
  if (!scan) return;
  const deep = await listRunningMonitorChecks(limit, scan);
  const checks = Array.from({ length: limit }, (_, i) => [
    newest[i],
    oldest[i],
    deep[i],
  ])
    .flat()
    .filter((check): check is (typeof deep)[number] => check !== undefined);
  const inspected = new Set<string>();
  let terminalAttempts = 0;
  try {
    for (const candidate of checks) {
      if (terminalAttempts >= limit) break;
      if (inspected.has(candidate.id)) continue;
      inspected.add(candidate.id);
      await reconcileMonitorCheckCandidate(candidate, () => {
        terminalAttempts++;
      });
    }
  } finally {
    let advanced = 0;
    for (const check of deep) {
      if (!inspected.has(check.id)) break;
      advanced++;
    }
    if (advanced === deep.length && deep.length < limit) {
      runningCheckScan = undefined;
    } else if (advanced > 0) {
      const last = deep[advanced - 1];
      runningCheckScan = {
        through: scan.through,
        after: { created_at: last.created_at, id: last.id },
      };
    }
  }
}

async function reconcileMonitorCheckCandidate(
  candidate: Awaited<ReturnType<typeof listRunningMonitorChecks>>[number],
  attempt: () => void,
): Promise<void> {
  const lockKey = `monitor-check-finalize:${candidate.id}`;
  const lockToken = uuidv7();
  const lock = await redisEvictConnection.set(
    lockKey,
    lockToken,
    "EX",
    60,
    "NX",
  );
  if (lock !== "OK") return;

  try {
    // The batch can outlive another finalizer. Read from the primary after
    // acquiring the lease rather than acting on that old running snapshot.
    const check = await getMonitorCheckForUpdate(
      candidate.team_id,
      candidate.monitor_id,
      candidate.id,
    );
    if (!check || check.status !== "running") return;

    // One org lookup per check — the billing service no longer makes it, so
    // the release, the stale-fail and the settle below all share this one.
    const orgId = await orgIdForTeam(check.team_id);

    const monitor = await getMonitorForUpdate(check.team_id, check.monitor_id);
    if (!monitor) {
      attempt();
      const failed = await updateMonitorCheckIfRunning(check.id, {
        status: "failed",
        finished_at: new Date().toISOString(),
        actual_credits: 0,
        error: "Monitor no longer exists.",
      });
      if (!failed) return;

      let released = true;
      if (failed.autumn_lock_id) {
        released = await autumnService
          .finalizeCreditsLock({
            lockId: failed.autumn_lock_id,
            action: "release",
            properties: {
              source: "monitorCheck",
              endpoint: "monitor",
              jobId: check.id,
            },
            team: orgId ? { teamId: check.team_id, orgId } : undefined,
          })
          .catch(error => {
            logger.warn(
              "Failed to release orphaned monitor check credit lock",
              {
                error,
                monitorId: check.monitor_id,
                checkId: check.id,
                lockId: failed.autumn_lock_id,
              },
            );
            return false;
          });
      }

      await updateMonitorCheck(failed.id, {
        billing_status: !failed.autumn_lock_id
          ? "not_applicable"
          : released
            ? "released"
            : "failed",
      });
      logger.warn("Failed orphaned monitor check", {
        monitorId: check.monitor_id,
        checkId: check.id,
      });
      return;
    }

    if (isMonitorCheckStale(check, new Date(), monitor.targets)) {
      attempt();
      await failStaleMonitorCheck({ monitor, check, orgId });
      return;
    }

    // The inline handler may still write target results after our primary read.
    let targetResults = Array.isArray(check.target_results)
      ? ([...check.target_results] as any[])
      : [];
    // True only when the persisted snapshot was empty and we rebuild the target
    // runs from recorded pages. That is the only case it is safe to write back
    // below: there is no live target_results to overwrite.
    const recoveredFromEmpty = targetResults.length === 0;
    if (recoveredFromEmpty) {
      targetResults = await recoverTargetRunsFromRecordedPages({
        monitor,
        check,
      });
    }

    await processRemovedPagesForCompletedCrawls({
      monitor,
      check,
      targetResults,
    });

    const unfinished = await findUnfinishedMonitorTarget(
      {
        ...check,
        target_results: targetResults,
      },
      monitor,
    );
    if (unfinished) {
      logger.info("Monitor check not ready to finalize", {
        monitorId: monitor.id,
        checkId: check.id,
        runningForMs: check.started_at
          ? Date.now() - Date.parse(check.started_at)
          : null,
        ...unfinished,
      });
      // Only persist target_results we recovered from an empty snapshot. Writing
      // back a non-empty stale snapshot here can DOWNGRADE a searchCompleted=true
      // that the inline handler persisted after this reconciler loaded its
      // snapshot, reverting the marker and stranding the check until the stale
      // reaper. The complete-path write below is safe: a search target can only
      // be complete once its snapshot already carries searchCompleted=true.
      if (recoveredFromEmpty && targetResults.length > 0) {
        await updateMonitorCheckIfRunning(check.id, {
          target_results: targetResults,
        });
      }
      return;
    }

    const [same, changed, newCount, removed, errorCount] = await Promise.all([
      countMonitorCheckPages({ checkId: check.id, status: "same" }),
      countMonitorCheckPages({ checkId: check.id, status: "changed" }),
      countMonitorCheckPages({ checkId: check.id, status: "new" }),
      countMonitorCheckPages({ checkId: check.id, status: "removed" }),
      countMonitorCheckPages({ checkId: check.id, status: "error" }),
    ]);
    const totalPages = same + changed + newCount + removed + errorCount;
    const actualCredits = await calculateMonitorCheckActualCredits({
      checkId: check.id,
      targets: monitor.targets,
      // Flat search credits come from target_results, not page metadata.
      targetResults,
    });

    // This conditional write is the durable claim. Even if the Redis lease
    // expires during preparation, only one worker may settle this check.
    attempt();
    const claimed = await updateMonitorCheckIfRunning(check.id, {
      status: errorCount > 0 ? "partial" : "completed",
      finished_at: new Date().toISOString(),
      actual_credits: actualCredits,
      total_pages: totalPages,
      same_count: same,
      changed_count: changed,
      new_count: newCount,
      removed_count: removed,
      error_count: errorCount,
      target_results: targetResults,
    });

    if (!claimed) return;
    let finalized = claimed;

    let settled = false;
    try {
      settled = await billMonitorCheck({
        monitor,
        check: finalized,
        actualCredits,
        lockId: claimed.autumn_lock_id,
        orgId,
      });
    } catch (error) {
      logger.warn("Failed to bill monitor check during reconciliation", {
        monitorId: monitor.id,
        checkId: finalized.id,
        error,
      });
    }

    // A refusal and a throw are the same fact — the settle did not land — and
    // both must be recorded as such. `firebillFinalize` answers `false`
    // without throwing, so the catch alone never saw them.
    if (claimed.autumn_lock_id) {
      if (!settled) {
        logger.error(
          "Monitor check settle did not land; the hold is unsettled and this run is unbilled",
          {
            monitorId: monitor.id,
            checkId: finalized.id,
            lockId: claimed.autumn_lock_id,
            actualCredits,
          },
        );
      }
      finalized = await updateMonitorCheck(check.id, {
        billing_status: settled ? "confirmed" : "failed",
      }).catch(updateError => {
        logger.warn("Failed to record monitor check billing outcome", {
          monitorId: monitor.id,
          checkId: finalized.id,
          error: updateError,
        });
        return finalized;
      });
    }

    if (await claimMonitorNotification(check.id)) {
      let notificationStatus: {
        webhook?: unknown;
        email?: unknown;
        slack?: unknown;
      } | null = null;
      try {
        const pages = (await listMonitorCheckPages({
          teamId: monitor.team_id,
          monitorId: monitor.id,
          checkId: check.id,
          limit: 100,
          skip: 0,
        })) as PageResult[];

        notificationStatus = await sendNotifications({
          monitor,
          check: finalized,
          pages,
        });

        finalized = await updateMonitorCheck(check.id, {
          notification_status: notificationStatus,
          webhook_payload: notificationStatus.webhook
            ? { summary: toSummaryObject(finalized) }
            : null,
          email_payload: notificationStatus.email
            ? { summary: toSummaryObject(finalized) }
            : null,
        });
      } catch (error) {
        logger.warn("Failed to send monitor check notifications", {
          monitorId: monitor.id,
          checkId: finalized.id,
          error,
        });
        notificationStatus = {
          webhook: {
            attempted: !!monitor.webhook,
            success: false,
            error: error instanceof Error ? error.message : String(error),
          },
          email: {
            attempted: !!monitor.notification?.email?.enabled,
            success: false,
            error: error instanceof Error ? error.message : String(error),
          },
        };
        finalized = await updateMonitorCheck(check.id, {
          notification_status: notificationStatus,
        }).catch(updateError => {
          logger.warn("Failed to record monitor check notification failure", {
            monitorId: monitor.id,
            checkId: finalized.id,
            error: updateError,
          });
          return finalized;
        });
      }
    }

    await updateMonitorScheduleAfterRun({
      monitor,
      check: finalized,
      summary: toSummaryObject(finalized),
    });

    logger.info("Reconciled monitor check", {
      monitorId: monitor.id,
      checkId: finalized.id,
      status: finalized.status,
      totalPages,
      same,
      changed,
      new: newCount,
      removed,
      errors: errorCount,
    });
  } catch (error) {
    logger.warn("Failed to reconcile monitor check", {
      error,
      checkId: candidate.id,
    });
  } finally {
    // An expired lease may already belong to another worker.
    await redisEvictConnection.eval(
      `if redis.call("get", KEYS[1]) == ARGV[1] then
      return redis.call("del", KEYS[1])
    end
    return 0`,
      1,
      lockKey,
      lockToken,
    );
  }
}

function toSummaryObject(check: MonitorCheckRow) {
  return {
    totalPages: check.total_pages,
    same: check.same_count,
    changed: check.changed_count,
    new: check.new_count,
    removed: check.removed_count,
    error: check.error_count,
  };
}