Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add monitor API orchestration (ENG-4857) (#3470)

Gergő Móriczbc3c57adbb26
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index 4e2306633..4d0e461d0 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;
// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configuration
export const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);
+export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;
export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;
export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;
export const HAS_PROXY = !!config.PROXY_SERVER;
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 88a8228a8..53c70e4db 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -437,7 +437,7 @@ describe("Crawl tests", () => {
async () => {
const res = await crawl(
{
- url: base,
+ url: new URL("/blog", base).href,
prompt:
"Crawl everything including external links and subdomains",
// Explicit options that should override the prompt
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index 840a2c7c7..1e5583f71 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -85,6 +85,96 @@ export async function scrapeWithFailure(
return raw.body;
}
+// =========================================
+// Monitor API
+// =========================================
+
+export type MonitorCreateInput = {
+ name: string;
+ schedule: { cron: string; timezone?: string };
+ webhook?: { url: string; headers?: Record<string, string> };
+ notification?: {
+ email?: {
+ enabled?: boolean;
+ recipients?: string[];
+ includeDiffs?: boolean;
+ };
+ };
+ targets: Array<
+ | {
+ type: "scrape";
+ urls: string[];
+ scrapeOptions?: Record<string, unknown>;
+ }
+ | {
+ type: "crawl";
+ url: string;
+ crawlOptions?: Record<string, unknown>;
+ scrapeOptions?: Record<string, unknown>;
+ }
+ >;
+ retentionDays?: number;
+};
+
+export async function monitorCreateRaw(
+ body: MonitorCreateInput,
+ identity: Identity,
+) {
+ return await request(TEST_API_URL)
+ .post("/v2/monitor")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body);
+}
+
+export async function monitorListRaw(identity: Identity) {
+ return await request(TEST_API_URL)
+ .get("/v2/monitor")
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorGetRaw(id: string, identity: Identity) {
+ return await request(TEST_API_URL)
+ .get(`/v2/monitor/${id}`)
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorPatchRaw(
+ id: string,
+ body: Partial<MonitorCreateInput> & { status?: "active" | "paused" },
+ identity: Identity,
+) {
+ return await request(TEST_API_URL)
+ .patch(`/v2/monitor/${id}`)
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send(body);
+}
+
+export async function monitorDeleteRaw(id: string, identity: Identity) {
+ return await request(TEST_API_URL)
+ .delete(`/v2/monitor/${id}`)
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorRunRaw(id: string, identity: Identity) {
+ return await request(TEST_API_URL)
+ .post(`/v2/monitor/${id}/run`)
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+}
+
+export async function monitorCheckRaw(
+ monitorId: string,
+ checkId: string,
+ identity: Identity,
+ query?: Record<string, string | number>,
+) {
+ const req = request(TEST_API_URL)
+ .get(`/v2/monitor/${monitorId}/checks/${checkId}`)
+ .set("Authorization", `Bearer ${identity.apiKey}`);
+ return query ? req.query(query) : req;
+}
+
export async function parseRaw(
body: {
options?: Omit<ParseRequestInput, "file">;
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.ts
new file mode 100644
index 000000000..9c677507a
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts
@@ -0,0 +1,142 @@
+import {
+ createTestIdUrl,
+ describeIf,
+ ALLOW_TEST_SUITE_WEBSITE,
+ TEST_SELF_HOST,
+} from "../lib";
+import {
+ idmux,
+ Identity,
+ monitorCheckRaw,
+ monitorCreateRaw,
+ monitorDeleteRaw,
+ monitorGetRaw,
+ monitorListRaw,
+ monitorPatchRaw,
+ monitorRunRaw,
+ scrapeTimeout,
+} from "./lib";
+
+describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "monitor",
+ concurrency: 20,
+ credits: 1000000,
+ });
+ }, 10000);
+
+ it("creates, lists, gets, pauses, and deletes a monitor", async () => {
+ const create = await monitorCreateRaw(
+ {
+ name: "snips monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl(), createTestIdUrl()],
+ scrapeOptions: { formats: ["markdown"] },
+ },
+ ],
+ notification: { email: { enabled: false } },
+ },
+ identity,
+ );
+
+ expect(create.statusCode).toBe(200);
+ expect(create.body.success).toBe(true);
+ expect(create.body.data.id).toEqual(expect.any(String));
+ expect(create.body.data.targets[0].id).toEqual(expect.any(String));
+
+ const id = create.body.data.id;
+ const list = await monitorListRaw(identity);
+ expect(list.statusCode).toBe(200);
+ expect(list.body.data.some((x: any) => x.id === id)).toBe(true);
+
+ const get = await monitorGetRaw(id, identity);
+ expect(get.statusCode).toBe(200);
+ expect(get.body.data.id).toBe(id);
+
+ const patch = await monitorPatchRaw(id, { status: "paused" }, identity);
+ expect(patch.statusCode).toBe(200);
+ expect(patch.body.data.status).toBe("paused");
+
+ const del = await monitorDeleteRaw(id, identity);
+ expect(del.statusCode).toBe(200);
+ expect(del.body.success).toBe(true);
+ });
+
+ it("rejects cron schedules under 15 minutes", async () => {
+ const response = await monitorCreateRaw(
+ {
+ name: "too frequent",
+ schedule: { cron: "*/5 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl()],
+ },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(400);
+ expect(response.body.success).toBe(false);
+ expect(response.body.error).toContain("15 minutes");
+ });
+
+ it(
+ "runs a manual scrape monitor check",
+ async () => {
+ const create = await monitorCreateRaw(
+ {
+ name: "manual monitor",
+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [createTestIdUrl(), createTestIdUrl()],
+ scrapeOptions: { formats: ["markdown"] },
+ },
+ ],
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+
+ const monitorId = create.body.data.id;
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ const checkId = run.body.id;
+
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, checkId, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (["completed", "partial", "failed"].includes(check.status)) break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+
+ expect(["completed", "partial"]).toContain(check.status);
+ expect(check.summary.totalPages).toBeGreaterThanOrEqual(2);
+ expect(check.pages.length).toBeGreaterThanOrEqual(1);
+ expect(check.next).toBeUndefined();
+
+ const firstPage = await monitorCheckRaw(monitorId, checkId, identity, {
+ limit: 1,
+ });
+ expect(firstPage.statusCode).toBe(200);
+ expect(firstPage.body.next).toContain("skip=1");
+ expect(firstPage.body.next).toContain("limit=1");
+ expect(firstPage.body.data.next).toBe(firstPage.body.next);
+ expect(firstPage.body.data.pages).toHaveLength(1);
+
+ await monitorDeleteRaw(monitorId, identity);
+ },
+ 2 * scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts b/apps/api/src/__tests__/snips/v2/scrape-query.test.ts
index 41cbd5b4d..b088edc1e 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-query.test.ts
@@ -1,4 +1,4 @@
-import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";
+import { concurrentIf, HAS_AI, HAS_FIREWORKS, TEST_PRODUCTION } from "../lib";
import {
scrape,
scrapeRaw,
@@ -61,7 +61,7 @@ describe("Query format", () => {
scrapeTimeout,
);
- concurrentIf(TEST_PRODUCTION || HAS_AI)(
+ concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(
"returns a direct quote answer when query mode is directQuote",
async () => {
const response = await scrape(
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 3ab18a905..7aa29e744 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -21,6 +21,11 @@ import {
SearchRequestInput,
toV2CrawlerOptions,
} from "../../../controllers/v2/types";
+import {
+ createMonitorSchema,
+ updateMonitorSchema,
+} from "../../../services/monitoring/types";
+import { getNextMonitorRunAt } from "../../../services/monitoring/cron";
describe("V2 Types Validation", () => {
describe("scrapeRequestSchema", () => {
@@ -1164,6 +1169,87 @@ describe("V2 Types Validation", () => {
});
});
+ describe("monitor schedules", () => {
+ it("should accept natural language schedule text", () => {
+ const result = createMonitorSchema.parse({
+ name: "Blog monitor",
+ schedule: {
+ text: "every 30 minutes",
+ },
+ targets: [
+ {
+ type: "scrape",
+ urls: ["https://example.com"],
+ },
+ ],
+ });
+
+ expect(result.schedule).toEqual({
+ cron: "*/30 * * * *",
+ timezone: "UTC",
+ });
+ });
+
+ it("should accept a natural language start minute", () => {
+ const result = updateMonitorSchema.parse({
+ schedule: {
+ text: "every 15 minutes starting at :07",
+ },
+ });
+
+ expect(result.schedule).toEqual({
+ cron: "7-59/15 * * * *",
+ timezone: "UTC",
+ });
+ });
+
+ it("should reject ambiguous schedule definitions", () => {
+ expect(() =>
+ updateMonitorSchema.parse({
+ schedule: {
+ cron: "*/30 * * * *",
+ text: "every 30 minutes",
+ },
+ }),
+ ).toThrow("Schedule must include either cron or text, not both");
+ });
+
+ it("should calculate next runs in the configured timezone", () => {
+ const next = getNextMonitorRunAt(
+ "0 9 * * *",
+ new Date("2026-01-01T13:00:00.000Z"),
+ "America/New_York",
+ );
+
+ expect(next.toISOString()).toBe("2026-01-01T14:00:00.000Z");
+ });
+
+ it("should accept monitor webhook event filters", () => {
+ const result = updateMonitorSchema.parse({
+ webhook: {
+ url: "https://example.com/webhook",
+ events: ["monitor.page", "monitor.check.completed"],
+ },
+ });
+
+ expect(result.webhook?.events).toEqual([
+ "monitor.page",
+ "monitor.check.completed",
+ ]);
+ });
+
+ it("should reject non-monitor webhook event filters", () => {
+ expect(() =>
+ updateMonitorSchema.parse({
+ webhook: {
+ url: "https://example.com/webhook",
+ events: ["completed"],
+ },
+ }),
+ ).toThrow();
+ });
+ });
+
describe("Edge cases", () => {
it("should handle URL without protocol (should add http://)", () => {
const input = {
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
new file mode 100644
index 000000000..914b43c85
--- /dev/null
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -0,0 +1,353 @@
+import { Response } from "express";
+import { z } from "zod";
+import { RequestWithAuth } from "./types";
+import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { getMonitorDiffArtifact } from "../../lib/gcs-monitoring";
+import {
+ createMonitorSchema,
+ listMonitorChecksQuerySchema,
+ listMonitorsQuerySchema,
+ monitorCheckDetailQuerySchema,
+ updateMonitorSchema,
+} from "../../services/monitoring/types";
+import {
+ createMonitor,
+ createMonitorCheck,
+ countMonitorCheckPages,
+ deleteMonitor,
+ estimateMonitorCreditsPerRun,
+ getMonitor,
+ getMonitorCheck,
+ getMonitorForUpdate,
+ listMonitorCheckPages,
+ listMonitorChecks,
+ listMonitors,
+ updateMonitor,
+} from "../../services/monitoring/store";
+import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";
+import {
+ estimateRunsPerMonth,
+ validateMonitorCron,
+} from "../../services/monitoring/cron";
+
+const monitorParamsSchema = z.strictObject({
+ monitorId: z.uuid(),
+});
+
+const monitorCheckParamsSchema = monitorParamsSchema.extend({
+ checkId: z.uuid(),
+});
+
+function rejectZdr(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+): boolean {
+ if (getScrapeZDR(req.acuc?.flags) === "forced") {
+ res.status(400).json({
+ success: false,
+ error:
+ "Monitoring requires retained snapshots and diffs, and is not supported for zero data retention teams.",
+ });
+ return true;
+ }
+ return false;
+}
+
+function serializeMonitor(monitor: any) {
+ return {
+ id: monitor.id,
+ name: monitor.name,
+ status: monitor.status,
+ schedule: {
+ cron: monitor.schedule_cron,
+ timezone: monitor.schedule_timezone,
+ },
+ nextRunAt: monitor.next_run_at,
+ lastRunAt: monitor.last_run_at,
+ currentCheckId: monitor.current_check_id,
+ targets: monitor.targets,
+ webhook: monitor.webhook,
+ notification: monitor.notification,
+ retentionDays: monitor.retention_days,
+ estimatedCreditsPerMonth: monitor.estimated_credits_per_month,
+ lastCheckSummary: monitor.last_check_summary,
+ createdAt: monitor.created_at,
+ updatedAt: monitor.updated_at,
+ };
+}
+
+function serializeCheck(check: any) {
+ return {
+ id: check.id,
+ monitorId: check.monitor_id,
+ status: check.status,
+ trigger: check.trigger,
+ scheduledFor: check.scheduled_for,
+ startedAt: check.started_at,
+ finishedAt: check.finished_at,
+ estimatedCredits: check.estimated_credits,
+ reservedCredits: check.reserved_credits,
+ actualCredits: check.actual_credits,
+ billingStatus: check.billing_status,
+ summary: {
+ totalPages: check.total_pages,
+ same: check.same_count,
+ changed: check.changed_count,
+ new: check.new_count,
+ removed: check.removed_count,
+ error: check.error_count,
+ },
+ targetResults: check.target_results,
+ notificationStatus: check.notification_status,
+ error: check.error,
+ createdAt: check.created_at,
+ updatedAt: check.updated_at,
+ };
+}
+
+export async function createMonitorController(
+ req: RequestWithAuth<{}, any, unknown>,
+ res: Response,
+) {
+ if (rejectZdr(req, res)) return;
+
+ const input = createMonitorSchema.parse(req.body);
+ let schedule;
+ try {
+ schedule = validateMonitorCron(
+ input.schedule.cron,
+ input.schedule.timezone,
+ );
+ } catch (error) {
+ return res.status(400).json({
+ success: false,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ const monitor = await createMonitor({
+ teamId: req.auth.team_id,
+ input,
+ nextRunAt: schedule.nextRunAt,
+ intervalMs: schedule.intervalMs,
+ });
+
+ res.status(200).json({
+ success: true,
+ data: serializeMonitor(monitor),
+ });
+}
+
+export async function listMonitorsController(
+ req: RequestWithAuth<{}, any, unknown>,
+ res: Response,
+) {
+ const query = listMonitorsQuerySchema.parse(req.query);
+ const monitors = await listMonitors({
+ teamId: req.auth.team_id,
+ limit: query.limit,
+ offset: query.offset,
+ });
+
+ res.status(200).json({
+ success: true,
+ data: monitors.map(serializeMonitor),
+ });
+}
+
+export async function getMonitorController(
+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,
+ res: Response,
+) {
+ const { monitorId } = monitorParamsSchema.parse(req.params);
+ const monitor = await getMonitor(req.auth.team_id, monitorId);
+ if (!monitor) {
+ return res.status(404).json({ success: false, error: "Monitor not found" });
+ }
+
+ res.status(200).json({
+ success: true,
+ data: serializeMonitor(monitor),
+ });
+}
+
+export async function updateMonitorController(
+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,
+ res: Response,
+) {
+ if (rejectZdr(req, res)) return;
+
+ const { monitorId } = monitorParamsSchema.parse(req.params);
+ const existing = await getMonitorForUpdate(req.auth.team_id, monitorId);
+ if (!existing) {
+ return res.status(404).json({ success: false, error: "Monitor not found" });
+ }
+
+ const input = updateMonitorSchema.parse(req.body);
+ const cron = input.schedule?.cron ?? existing.schedule_cron;
+ const timezone = input.schedule?.timezone ?? existing.schedule_timezone;
+ let schedule;
+ try {
+ schedule = validateMonitorCron(cron, timezone);
+ } catch (error) {
+ return res.status(400).json({
+ success: false,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ }
+ const monitor = await updateMonitor({
+ teamId: req.auth.team_id,
+ monitorId,
+ input,
+ nextRunAt: input.schedule ? schedule.nextRunAt : undefined,
+ intervalMs:
+ input.schedule || input.targets ? schedule.intervalMs : undefined,
+ });
+
+ res.status(200).json({
+ success: true,
+ data: serializeMonitor(monitor),
+ });
+}
+
+export async function deleteMonitorController(
+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,
+ res: Response,
+) {
+ const { monitorId } = monitorParamsSchema.parse(req.params);
+ const deleted = await deleteMonitor({
+ teamId: req.auth.team_id,
+ monitorId,
+ });
+ if (!deleted) {
+ return res.status(404).json({ success: false, error: "Monitor not found" });
+ }
+
+ res.status(200).json({ success: true });
+}
+
+export async function runMonitorController(
+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,
+ res: Response,
+) {
+ if (rejectZdr(req, res)) return;
+
+ const { monitorId } = monitorParamsSchema.parse(req.params);
+ const monitor = await getMonitorForUpdate(req.auth.team_id, monitorId);
+ if (!monitor) {
+ return res.status(404).json({ success: false, error: "Monitor not found" });
+ }
+ if (monitor.current_check_id) {
+ return res.status(409).json({
+ success: false,
+ error: "Monitor check is already running.",
+ checkId: monitor.current_check_id,
+ });
+ }
+
+ const check = await createMonitorCheck({
+ monitor,
+ trigger: "manual",
+ });
+ await enqueueMonitorCheck({
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ });
+
+ res.status(200).json({
+ success: true,
+ id: check.id,
+ data: serializeCheck(check),
+ });
+}
+
+export async function listMonitorChecksController(
+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,
+ res: Response,
+) {
+ const { monitorId } = monitorParamsSchema.parse(req.params);
+ const monitor = await getMonitor(req.auth.team_id, monitorId);
+ if (!monitor) {
+ return res.status(404).json({ success: false, error: "Monitor not found" });
+ }
+
+ const query = listMonitorChecksQuerySchema.parse(req.query);
+ const checks = await listMonitorChecks({
+ teamId: req.auth.team_id,
+ monitorId,
+ limit: query.limit,
+ offset: query.offset,
+ status: query.status,
+ });
+
+ res.status(200).json({
+ success: true,
+ data: checks.map(serializeCheck),
+ });
+}
+
+export async function getMonitorCheckController(
+ req: RequestWithAuth<{ monitorId: string; checkId: string }, any, unknown>,
+ res: Response,
+) {
+ const { monitorId, checkId } = monitorCheckParamsSchema.parse(req.params);
+ const query = monitorCheckDetailQuerySchema.parse(req.query);
+ const skip = query.skip;
+ const check = await getMonitorCheck(req.auth.team_id, monitorId, checkId);
+ if (!check) {
+ return res.status(404).json({ success: false, error: "Check not found" });
+ }
+
+ const [pages, totalPagesForFilter] = await Promise.all([
+ listMonitorCheckPages({
+ teamId: req.auth.team_id,
+ monitorId,
+ checkId,
+ limit: query.limit,
+ skip,
+ status: query.status,
+ }),
+ countMonitorCheckPages({
+ checkId,
+ status: query.status,
+ }),
+ ]);
+
+ const pagesWithDiffs = await Promise.all(
+ pages.map(async page => ({
+ id: page.id,
+ targetId: page.target_id,
+ url: page.url,
+ status: page.status,
+ previousScrapeId: page.previous_scrape_id,
+ currentScrapeId: page.current_scrape_id,
+ statusCode: page.status_code,
+ error: page.error,
+ metadata: page.metadata,
+ diff: await getMonitorDiffArtifact(page.diff_gcs_key),
+ createdAt: page.created_at,
+ })),
+ );
+ const nextSkip = skip + pagesWithDiffs.length;
+ const next = (() => {
+ if (totalPagesForFilter <= nextSkip) return undefined;
+ const url = new URL(
+ `/v2/monitor/${monitorId}/checks/${checkId}`,
+ `${req.protocol}://${req.get("host")}`,
+ );
+ url.searchParams.set("skip", String(nextSkip));
+ url.searchParams.set("limit", String(query.limit));
+ if (query.status) url.searchParams.set("status", query.status);
+ return url.toString();
+ })();
+
+ res.status(200).json({
+ success: true,
+ next,
+ data: {
+ ...serializeCheck(check),
+ pages: pagesWithDiffs,
+ next,
+ },
+ });
+}
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 1deef110a..51e5c1822 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -1,5 +1,6 @@
import { config } from "./config";
import { type ChildProcess, spawn } from "child_process";
+import { existsSync } from "fs";
import * as net from "net";
import { basename, join } from "path";
import { HTML_TO_MARKDOWN_PATH } from "./natives";
@@ -19,9 +20,15 @@ let nuqRabbitMQContainer: {
containerRuntime: string;
} | null = null;
-// Get the monorepo root (apps/api/dist/src -> ../../../..)
+// Get the monorepo root for both tsx source execution and compiled dist execution.
// __dirname is available in CommonJS (which this compiles to)
-const MONOREPO_ROOT = join(__dirname, "..", "..", "..", "..");
+const SOURCE_MONOREPO_ROOT = join(__dirname, "..", "..", "..");
+const DIST_MONOREPO_ROOT = join(__dirname, "..", "..", "..", "..");
+const MONOREPO_ROOT = existsSync(
+ join(SOURCE_MONOREPO_ROOT, "apps", "nuq-postgres"),
+)
+ ? SOURCE_MONOREPO_ROOT
+ : DIST_MONOREPO_ROOT;
const NUQ_POSTGRES_PATH = join(MONOREPO_ROOT, "apps", "nuq-postgres");
interface ProcessResult {
diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.ts
new file mode 100644
index 000000000..36b3e9e12
--- /dev/null
+++ b/apps/api/src/lib/gcs-monitoring.ts
@@ -0,0 +1,64 @@
+import { config } from "../config";
+import { storage } from "./gcs-jobs";
+
+type MonitorDiffArtifact = {
+ url: string;
+ previousScrapeId: string | null;
+ currentScrapeId: string | null;
+ text: string;
+ json: unknown;
+ generatedAt: string;
+};
+
+const contentType = "application/json";
+
+export function monitorDiffGcsKey(params: {
+ teamId: string;
+ monitorId: string;
+ checkId: string;
+ pageId: string;
+}): string {
+ return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;
+}
+
+export async function saveMonitorDiffArtifact(
+ key: string,
+ artifact: MonitorDiffArtifact,
+): Promise<{ textBytes: number; jsonBytes: number }> {
+ const payload = JSON.stringify(artifact);
+ if (!config.GCS_BUCKET_NAME) {
+ return {
+ textBytes: Buffer.byteLength(artifact.text),
+ jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
+ };
+ }
+
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
+ await bucket.file(key).save(payload, {
+ contentType,
+ resumable: false,
+ });
+
+ return {
+ textBytes: Buffer.byteLength(artifact.text),
+ jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),
+ };
+}
+
+export async function getMonitorDiffArtifact(
+ key: string | null | undefined,
+): Promise<MonitorDiffArtifact | null> {
+ if (!key || !config.GCS_BUCKET_NAME) return null;
+
+ const bucket = storage.bucket(config.GCS_BUCKET_NAME);
+ try {
+ const [contents] = await bucket.file(key).download();
+ return JSON.parse(contents.toString()) as MonitorDiffArtifact;
+ } catch (error) {
+ const maybeGcsError = error as { code?: number; statusCode?: number };
+ if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {
+ return null;
+ }
+ throw error;
+ }
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index d84e30221..407e1f2de 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -65,6 +65,16 @@ import {
scrapeInteractController,
scrapeStopInteractiveBrowserController,
} from "../controllers/v2/scrape-browser";
+import {
+ createMonitorController,
+ deleteMonitorController,
+ getMonitorCheckController,
+ getMonitorController,
+ listMonitorChecksController,
+ listMonitorsController,
+ runMonitorController,
+ updateMonitorController,
+} from "../controllers/v2/monitor";
expressWs(express());
@@ -448,6 +458,63 @@ v2Router.get(
wrap(activityController),
);
+v2Router.post(
+ "/monitor",
+ authMiddleware(RateLimiterMode.Crawl),
+ countryCheck,
+ checkCreditsMiddleware(1),
+ blocklistMiddleware,
+ wrap(createMonitorController),
+);
+
+v2Router.get(
+ "/monitor",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(listMonitorsController),
+);
+
+v2Router.get(
+ "/monitor/:monitorId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(getMonitorController),
+);
+
+v2Router.patch(
+ "/monitor/:monitorId",
+ authMiddleware(RateLimiterMode.Crawl),
+ countryCheck,
+ checkCreditsMiddleware(1),
+ blocklistMiddleware,
+ wrap(updateMonitorController),
+);
+
+v2Router.delete(
+ "/monitor/:monitorId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(deleteMonitorController),
+);
+
+v2Router.post(
+ "/monitor/:monitorId/run",
+ authMiddleware(RateLimiterMode.Crawl),
+ countryCheck,
+ checkCreditsMiddleware(1),
+ blocklistMiddleware,
+ wrap(runMonitorController),
+);
+
+v2Router.get(
+ "/monitor/:monitorId/checks",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(listMonitorChecksController),
+);
+
+v2Router.get(
+ "/monitor/:monitorId/checks/:checkId",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(getMonitorCheckController),
+);
+
v2Router.post(
"/browser",
authMiddleware(RateLimiterMode.Browser),
diff --git a/apps/api/src/services/billing/types.ts b/apps/api/src/services/billing/types.ts
index e39dfb449..87ae1ec54 100644
--- a/apps/api/src/services/billing/types.ts
+++ b/apps/api/src/services/billing/types.ts
@@ -9,6 +9,7 @@ export type BillingEndpoint =
| "interact"
| "llms_txt"
| "map"
+ | "monitor"
| "parse"
| "scrape"
| "search";
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 1a6a11e01..5241a6d37 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -222,6 +222,8 @@ export type LoggedScrape = {
skipNuq: boolean;
zeroDataRetention: boolean;
is_parse?: boolean;
+ monitor_id?: string | null;
+ monitor_check_id?: string | null;
};
export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
@@ -259,6 +261,12 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
? null
: (scrape.pdf_num_pages ?? null),
credits_cost: scrape.credits_cost,
+ ...(scrape.is_parse
+ ? {}
+ : {
+ monitor_id: scrape.monitor_id ?? null,
+ monitor_check_id: scrape.monitor_check_id ?? null,
+ }),
},
force,
logger,
@@ -322,6 +330,8 @@ type LoggedCrawl = {
credits_cost: number;
zeroDataRetention: boolean;
cancelled: boolean;
+ monitor_id?: string | null;
+ monitor_check_id?: string | null;
};
export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {
@@ -350,6 +360,8 @@ export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {
num_docs: crawl.num_docs,
credits_cost: crawl.credits_cost,
cancelled: crawl.cancelled,
+ monitor_id: crawl.monitor_id ?? null,
+ monitor_check_id: crawl.monitor_check_id ?? null,
},
force,
logger,
diff --git a/apps/api/src/services/monitoring/cron.ts b/apps/api/src/services/monitoring/cron.ts
new file mode 100644
index 000000000..cc4313a16
--- /dev/null
+++ b/apps/api/src/services/monitoring/cron.ts
@@ -0,0 +1,247 @@
+const MIN_MONITOR_INTERVAL_MS = 15 * 60 * 1000;
+const SEARCH_LIMIT_MINUTES = 366 * 24 * 60;
+
+type CronField = Set<number>;
+type CronSpec = ReturnType<typeof parseCron>;
+
+const WEEKDAY_TO_NUMBER: Record<string, number> = {
+ sun: 0,
+ mon: 1,
+ tue: 2,
+ wed: 3,
+ thu: 4,
+ fri: 5,
+ sat: 6,
+};
+
+function parseMinuteStart(value: string | undefined): number {
+ if (value === undefined) return 0;
+ const minute = Number(value);
+ if (!Number.isInteger(minute) || minute < 0 || minute > 59) {
+ throw new Error("Schedule start minute must be between 0 and 59");
+ }
+ return minute;
+}
+
+function assertDivides(value: number, divisor: number, unit: string) {
+ if (divisor % value !== 0) {
+ throw new Error(`${unit} interval must divide evenly into ${divisor}`);
+ }
+}
+
+export function parseMonitorScheduleText(input: string): string {
+ const text = input
+ .trim()
+ .toLowerCase()
+ .replace(/[.,]/g, "")
+ .replace(/\s+/g, " ");
+
+ const minuteMatch = text.match(
+ /^every (\d+) (?:minutes?|mins?)(?: (?:starting|start)(?: at| on)? :?(\d{1,2}))?$/,
+ );
+ if (minuteMatch) {
+ const interval = Number(minuteMatch[1]);
+ if (!Number.isInteger(interval) || interval <= 0 || interval > 60) {
+ throw new Error("Minute interval must be between 1 and 60");
+ }
+ assertDivides(interval, 60, "Minute");
+ const startMinute = parseMinuteStart(minuteMatch[2]);
+ return startMinute === 0
+ ? `*/${interval} * * * *`
+ : `${startMinute}-59/${interval} * * * *`;
+ }
+
+ const hourlyMatch = text.match(
+ /^(?:hourly|every hour)(?: (?:at|starting at) :?(\d{1,2}))?$/,
+ );
+ if (hourlyMatch) {
+ return `${parseMinuteStart(hourlyMatch[1])} * * * *`;
+ }
+
+ const hourMatch = text.match(/^every (\d+) (?:hours?|hrs?)$/);
+ if (hourMatch) {
+ const interval = Number(hourMatch[1]);
+ if (!Number.isInteger(interval) || interval <= 0 || interval > 24) {
+ throw new Error("Hour interval must be between 1 and 24");
+ }
+ assertDivides(interval, 24, "Hour");
+ return `0 */${interval} * * *`;
+ }
+
+ const dailyMatch = text.match(
+ /^(?:daily|every day)(?: at (\d{1,2})(?::(\d{2}))?)?$/,
+ );
+ if (dailyMatch) {
+ const hour = dailyMatch[1] === undefined ? 0 : Number(dailyMatch[1]);
+ const minute = dailyMatch[2] === undefined ? 0 : Number(dailyMatch[2]);
+ if (!Number.isInteger(hour) || hour < 0 || hour > 23) {
+ throw new Error("Daily schedule hour must be between 0 and 23");
+ }
+ if (!Number.isInteger(minute) || minute < 0 || minute > 59) {
+ throw new Error("Daily schedule minute must be between 0 and 59");
+ }
+ return `${minute} ${hour} * * *`;
+ }
+
+ if (text === "weekly" || text === "every week") {
+ return "0 0 * * 0";
+ }
+
+ throw new Error(
+ "Unsupported schedule text. Try phrases like 'every 30 minutes', 'hourly', or 'daily at 9:00'",
+ );
+}
+
+function parseField(field: string, min: number, max: number): CronField {
+ const values = new Set<number>();
+ for (const part of field.split(",")) {
+ const [rangePart, stepPart] = part.split("/");
+ const step = stepPart === undefined ? 1 : Number(stepPart);
+ if (!Number.isInteger(step) || step <= 0) {
+ throw new Error("Invalid cron step");
+ }
+
+ let start: number;
+ let end: number;
+ if (rangePart === "*") {
+ start = min;
+ end = max;
+ } else if (rangePart.includes("-")) {
+ const [a, b] = rangePart.split("-").map(Number);
+ start = a;
+ end = b;
+ } else {
+ start = Number(rangePart);
+ end = start;
+ }
+
+ if (
+ !Number.isInteger(start) ||
+ !Number.isInteger(end) ||
+ start < min ||
+ end > max ||
+ start > end
+ ) {
+ throw new Error("Invalid cron field");
+ }
+
+ for (let value = start; value <= end; value += step) {
+ values.add(value);
+ }
+ }
+ return values;
+}
+
+function parseDayOfWeek(field: string): CronField {
+ const values = parseField(field, 0, 7);
+ if (values.has(7)) {
+ values.add(0);
+ values.delete(7);
+ }
+ return values;
+}
+
+function parseCron(cron: string) {
+ const parts = cron.trim().split(/\s+/);
+ if (parts.length !== 5) {
+ throw new Error("Cron expression must contain five fields");
+ }
+
+ return {
+ minutes: parseField(parts[0], 0, 59),
+ hours: parseField(parts[1], 0, 23),
+ daysOfMonth: parseField(parts[2], 1, 31),
+ months: parseField(parts[3], 1, 12),
+ daysOfWeek: parseDayOfWeek(parts[4]),
+ };
+}
+
+function validateTimeZone(timeZone: string): void {
+ try {
+ new Intl.DateTimeFormat("en-US", { timeZone }).format(new Date());
+ } catch {
+ throw new Error(`Invalid monitor schedule timezone: ${timeZone}`);
+ }
+}
+
+function getZonedParts(date: Date, timeZone: string) {
+ const parts = new Intl.DateTimeFormat("en-US", {
+ timeZone,
+ hourCycle: "h23",
+ minute: "2-digit",
+ hour: "2-digit",
+ day: "2-digit",
+ month: "2-digit",
+ weekday: "short",
+ }).formatToParts(date);
+
+ const values = Object.fromEntries(
+ parts
+ .filter(part => part.type !== "literal")
+ .map(part => [part.type, part.value]),
+ );
+
+ return {
+ minutes: Number(values.minute),
+ hours: Number(values.hour),
+ daysOfMonth: Number(values.day),
+ months: Number(values.month),
+ daysOfWeek: WEEKDAY_TO_NUMBER[String(values.weekday).toLowerCase()],
+ };
+}
+
+function matches(date: Date, cron: CronSpec, timeZone: string): boolean {
+ const zoned = getZonedParts(date, timeZone);
+ return (
+ cron.minutes.has(zoned.minutes) &&
+ cron.hours.has(zoned.hours) &&
+ cron.daysOfMonth.has(zoned.daysOfMonth) &&
+ cron.months.has(zoned.months) &&
+ cron.daysOfWeek.has(zoned.daysOfWeek)
+ );
+}
+
+export function getNextMonitorRunAt(
+ cronExpression: string,
+ from = new Date(),
+ timeZone = "UTC",
+): Date {
+ validateTimeZone(timeZone);
+ const cron = parseCron(cronExpression);
+ const candidate = new Date(from);
+ candidate.setUTCSeconds(0, 0);
+ candidate.setUTCMinutes(candidate.getUTCMinutes() + 1);
+
+ for (let i = 0; i < SEARCH_LIMIT_MINUTES; i++) {
+ if (matches(candidate, cron, timeZone)) {
+ return new Date(candidate);
+ }
+ candidate.setUTCMinutes(candidate.getUTCMinutes() + 1);
+ }
+
+ throw new Error("Cron expression did not produce a run within one year");
+}
+
+export function validateMonitorCron(
+ cronExpression: string,
+ timeZone = "UTC",
+): {
+ nextRunAt: Date;
+ intervalMs: number;
+} {
+ const nextRunAt = getNextMonitorRunAt(cronExpression, new Date(), timeZone);
+ const secondRunAt = getNextMonitorRunAt(cronExpression, nextRunAt, timeZone);
+ const intervalMs = secondRunAt.getTime() - nextRunAt.getTime();
+ if (intervalMs < MIN_MONITOR_INTERVAL_MS) {
+ throw new Error(
+ "Monitor schedule must not run more often than every 15 minutes",
+ );
+ }
+
+ return { nextRunAt, intervalMs };
+}
+
+export function estimateRunsPerMonth(intervalMs: number): number {
+ const daysPerMonth = 30;
+ return Math.ceil((daysPerMonth * 24 * 60 * 60 * 1000) / intervalMs);
+}
diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.ts
new file mode 100644
index 000000000..3197367d3
--- /dev/null
+++ b/apps/api/src/services/monitoring/diff.ts
@@ -0,0 +1,105 @@
+import gitDiff from "git-diff";
+import parseDiff from "parse-diff";
+
+type MonitoringDiffResult =
+ | {
+ status: "same";
+ text?: undefined;
+ json?: undefined;
+ }
+ | {
+ status: "changed";
+ text: string;
+ json: {
+ files: Array<{
+ from: string | null;
+ to: string | null;
+ chunks: Array<{
+ content: string;
+ changes: Array<{
+ type: string;
+ normal?: boolean;
+ add?: boolean;
+ del?: boolean;
+ ln?: number;
+ ln1?: number;
+ ln2?: number;
+ content: string;
+ }>;
+ }>;
+ }>;
+ };
+ };
+
+function normalizeMarkdownForChangeTracking(markdown: string): string {
+ return [...markdown.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")]
+ .sort()
+ .join("");
+}
+
+export function diffMonitorMarkdown(
+ previousMarkdown: string,
+ currentMarkdown: string,
+): MonitoringDiffResult {
+ if (
+ normalizeMarkdownForChangeTracking(previousMarkdown) ===
+ normalizeMarkdownForChangeTracking(currentMarkdown)
+ ) {
+ return { status: "same" };
+ }
+
+ const text = gitDiff(previousMarkdown, currentMarkdown, {
+ color: false,
+ wordDiff: false,
+ });
+ const structured = parseDiff(text);
+
+ return {
+ status: "changed",
+ text,
+ json: {
+ files: structured.map(file => ({
+ from: file.from || null,
+ to: file.to || null,
+ chunks: file.chunks.map(chunk => ({
+ content: chunk.content,
+ changes: chunk.changes.map(change => {
+ const base = {
+ type: change.type,
+ content: change.content,
+ };
+
+ if (
+ change.type === "normal" &&
+ "ln1" in change &&
+ "ln2" in change
+ ) {
+ return {
+ ...base,
+ normal: true,
+ ln1: change.ln1,
+ ln2: change.ln2,
+ };
+ }
+ if (change.type === "add" && "ln" in change) {
+ return {
+ ...base,
+ add: true,
+ ln: change.ln,
+ };
+ }
+ if (change.type === "del" && "ln" in change) {
+ return {
+ ...base,
+ del: true,
+ ln: change.ln,
+ };
+ }
+
+ return base;
+ }),
+ })),
+ })),
+ },
+ };
+}
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
new file mode 100644
index 000000000..77cc8675f
--- /dev/null
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -0,0 +1,133 @@
+import amqp from "amqplib";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+
+const MONITOR_CHECK_QUEUE = "monitor.checks";
+const MONITOR_CHECK_DLX = "monitor.checks.dlx";
+const MONITOR_CHECK_DLQ = "monitor.checks.dlq";
+
+const logger = _logger.child({ module: "monitoring-queue" });
+
+export type MonitorCheckJobData = {
+ monitorId: string;
+ checkId: string;
+ teamId: string;
+};
+
+let connection: amqp.ChannelModel | null = null;
+let channel: amqp.Channel | null = null;
+
+async function getChannel(): Promise<amqp.Channel> {
+ if (channel) return channel;
+
+ const url = config.NUQ_RABBITMQ_URL;
+ if (!url) {
+ throw new Error("NUQ_RABBITMQ_URL is not configured");
+ }
+
+ connection = await amqp.connect(url);
+ channel = await connection.createChannel();
+
+ await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });
+ await channel.assertQueue(MONITOR_CHECK_DLQ, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ },
+ });
+ await channel.bindQueue(
+ MONITOR_CHECK_DLQ,
+ MONITOR_CHECK_DLX,
+ MONITOR_CHECK_QUEUE,
+ );
+
+ await channel.assertQueue(MONITOR_CHECK_QUEUE, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-dead-letter-exchange": MONITOR_CHECK_DLX,
+ "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,
+ "x-delivery-limit": 1,
+ },
+ });
+
+ connection.on("close", () => {
+ logger.warn("Monitor queue connection closed");
+ connection = null;
+ channel = null;
+ });
+
+ connection.on("error", error => {
+ logger.error("Monitor queue connection error", { error });
+ });
+
+ return channel;
+}
+
+export async function addMonitorCheckJob(
+ data: MonitorCheckJobData,
+): Promise<void> {
+ const ch = await getChannel();
+ const sent = ch.sendToQueue(
+ MONITOR_CHECK_QUEUE,
+ Buffer.from(JSON.stringify(data)),
+ {
+ persistent: true,
+ contentType: "application/json",
+ messageId: data.checkId,
+ },
+ );
+
+ if (!sent) {
+ logger.warn("Monitor check message buffer full", {
+ monitorId: data.monitorId,
+ checkId: data.checkId,
+ });
+ }
+
+ logger.info("Monitor check job added to queue", {
+ monitorId: data.monitorId,
+ checkId: data.checkId,
+ teamId: data.teamId,
+ });
+}
+
+export async function consumeMonitorCheckJobs(
+ handler: (data: MonitorCheckJobData) => Promise<void>,
+): Promise<void> {
+ const ch = await getChannel();
+ await ch.prefetch(1);
+
+ await ch.consume(
+ MONITOR_CHECK_QUEUE,
+ async msg => {
+ if (!msg) return;
+
+ let data: MonitorCheckJobData;
+ try {
+ data = JSON.parse(msg.content.toString()) as MonitorCheckJobData;
+ } catch (error) {
+ logger.error("Failed to parse monitor check job", { error });
+ ch.nack(msg, false, false);
+ return;
+ }
+
+ const jobLogger = logger.child({
+ monitorId: data.monitorId,
+ checkId: data.checkId,
+ teamId: data.teamId,
+ });
+
+ try {
+ await handler(data);
+ ch.ack(msg);
+ } catch (error) {
+ jobLogger.error("Monitor check job failed", { error });
+ ch.nack(msg, false, false);
+ }
+ },
+ { noAck: false },
+ );
+
+ logger.info("Started consuming monitor check jobs");
+}
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
new file mode 100644
index 000000000..2c06706f1
--- /dev/null
+++ b/apps/api/src/services/monitoring/results.ts
@@ -0,0 +1,227 @@
+import { v7 as uuidv7 } from "uuid";
+import { NuQJob } from "../worker/nuq";
+import { ScrapeJobData } from "../../types";
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+import {
+ monitorDiffGcsKey,
+ saveMonitorDiffArtifact,
+} from "../../lib/gcs-monitoring";
+import { logger as _logger } from "../../lib/logger";
+import { createWebhookSender, WebhookEvent } from "../webhook";
+import { diffMonitorMarkdown } from "./diff";
+import {
+ getMonitorForUpdate,
+ getMonitorPage,
+ hashMonitorUrl,
+ insertMonitorCheckPages,
+ upsertMonitorPage,
+} from "./store";
+
+const logger = _logger.child({ module: "monitoring-results" });
+
+function getDocumentUrl(doc: any, fallback: string): string {
+ return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;
+}
+
+function getDocumentStatusCode(doc: any): number | null {
+ return typeof doc?.metadata?.statusCode === "number"
+ ? doc.metadata.statusCode
+ : null;
+}
+
+async function sendMonitorPageWebhook(params: {
+ teamId: string;
+ monitorId: string;
+ checkId: string;
+ url: string;
+ status: string;
+ previousScrapeId?: string | null;
+ currentScrapeId?: string | null;
+ error?: string | null;
+}) {
+ try {
+ const monitor = await getMonitorForUpdate(params.teamId, params.monitorId);
+ if (!monitor?.webhook) return;
+
+ const sender = await createWebhookSender({
+ teamId: params.teamId,
+ jobId: params.checkId,
+ webhook: monitor.webhook as any,
+ v0: false,
+ });
+
+ await sender?.send(WebhookEvent.MONITOR_PAGE, {
+ success: params.status !== "error",
+ data: {
+ monitorId: params.monitorId,
+ checkId: params.checkId,
+ url: params.url,
+ status: params.status,
+ previousScrapeId: params.previousScrapeId ?? null,
+ currentScrapeId: params.currentScrapeId ?? null,
+ error: params.error ?? null,
+ },
+ error: params.error ?? undefined,
+ });
+ } catch (error) {
+ logger.warn("Failed to send monitor page webhook", {
+ error,
+ monitorId: params.monitorId,
+ checkId: params.checkId,
+ url: params.url,
+ status: params.status,
+ });
+ }
+}
+
+export async function recordMonitorScrapeSuccess(
+ job: NuQJob<ScrapeJobData>,
+ doc: any,
+): Promise<void> {
+ const monitoring = job.data.monitoring;
+ if (!monitoring || job.data.mode !== "single_urls") return;
+
+ const url = getDocumentUrl(doc, job.data.url);
+ const previous = await getMonitorPage({
+ monitorId: monitoring.monitorId,
+ targetId: monitoring.targetId,
+ url,
+ });
+
+ let status: "same" | "new" | "changed" = "new";
+ let diffGcsKey: string | null = null;
+ let diffTextBytes: number | null = null;
+ let diffJsonBytes: number | null = null;
+
+ if (previous?.last_scrape_id && !previous.is_removed) {
+ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
+ const previousMarkdown = previousDoc?.markdown;
+ const currentMarkdown = doc?.markdown;
+
+ if (previousMarkdown && currentMarkdown) {
+ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
+ status = diff.status;
+
+ if (diff.status === "changed") {
+ diffGcsKey = monitorDiffGcsKey({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ pageId: uuidv7(),
+ });
+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, {
+ url,
+ previousScrapeId: previous.last_scrape_id,
+ currentScrapeId: job.id,
+ text: diff.text,
+ json: diff.json,
+ generatedAt: new Date().toISOString(),
+ });
+ diffTextBytes = sizes.textBytes;
+ diffJsonBytes = sizes.jsonBytes;
+ }
+ } else {
+ status = "changed";
+ }
+ }
+
+ await upsertMonitorPage({
+ monitorId: monitoring.monitorId,
+ teamId: job.data.team_id,
+ targetId: monitoring.targetId,
+ url,
+ source: monitoring.source,
+ checkId: monitoring.checkId,
+ scrapeId: job.id,
+ status,
+ metadata: {
+ title: doc?.metadata?.title ?? null,
+ statusCode: getDocumentStatusCode(doc),
+ creditsUsed: doc?.metadata?.creditsUsed ?? null,
+ },
+ });
+
+ await insertMonitorCheckPages([
+ {
+ check_id: monitoring.checkId,
+ monitor_id: monitoring.monitorId,
+ team_id: job.data.team_id,
+ target_id: monitoring.targetId,
+ url,
+ url_hash: hashMonitorUrl(url),
+ status,
+ previous_scrape_id: previous?.last_scrape_id ?? null,
+ current_scrape_id: job.id,
+ diff_gcs_key: diffGcsKey,
+ diff_text_bytes: diffTextBytes,
+ diff_json_bytes: diffJsonBytes,
+ status_code: getDocumentStatusCode(doc),
+ metadata: {
+ title: doc?.metadata?.title ?? null,
+ creditsUsed: doc?.metadata?.creditsUsed ?? null,
+ },
+ },
+ ]);
+
+ logger.info("Recorded monitor scrape result", {
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ targetId: monitoring.targetId,
+ scrapeId: job.id,
+ url,
+ status,
+ previousScrapeId: previous?.last_scrape_id ?? null,
+ diffGcsKey,
+ });
+
+ await sendMonitorPageWebhook({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ url,
+ status,
+ previousScrapeId: previous?.last_scrape_id ?? null,
+ currentScrapeId: job.id,
+ });
+}
+
+export async function recordMonitorScrapeFailure(
+ job: NuQJob<ScrapeJobData>,
+ error: unknown,
+): Promise<void> {
+ const monitoring = job.data.monitoring;
+ if (!monitoring || job.data.mode !== "single_urls") return;
+
+ await insertMonitorCheckPages([
+ {
+ check_id: monitoring.checkId,
+ monitor_id: monitoring.monitorId,
+ team_id: job.data.team_id,
+ target_id: monitoring.targetId,
+ url: job.data.url,
+ url_hash: hashMonitorUrl(job.data.url),
+ status: "error",
+ current_scrape_id: job.id,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ ]);
+
+ logger.info("Recorded monitor scrape failure", {
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ targetId: monitoring.targetId,
+ scrapeId: job.id,
+ url: job.data.url,
+ error: error instanceof Error ? error.message : String(error),
+ });
+
+ await sendMonitorPageWebhook({
+ teamId: job.data.team_id,
+ monitorId: monitoring.monitorId,
+ checkId: monitoring.checkId,
+ url: job.data.url,
+ status: "error",
+ currentScrapeId: job.id,
+ error: error instanceof Error ? error.message : String(error),
+ });
+}
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
new file mode 100644
index 000000000..4afa1ce73
--- /dev/null
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -0,0 +1,1098 @@
+import { v7 as uuidv7 } from "uuid";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+import { logRequest } from "../logging/log_job";
+import { processJobInternal } from "../worker/scrape-worker";
+import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";
+import { ScrapeJobData } from "../../types";
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+import {
+ monitorDiffGcsKey,
+ saveMonitorDiffArtifact,
+} from "../../lib/gcs-monitoring";
+import { diffMonitorMarkdown } from "./diff";
+import { autumnService } from "../autumn/autumn.service";
+import { getBillingQueue } from "../queue-service";
+import {
+ crawlToCrawler,
+ markCrawlActive,
+ saveCrawl,
+ StoredCrawl,
+} from "../../lib/crawl-redis";
+import { _addScrapeJobToBullMQ, addScrapeJob } from "../queue-jobs";
+import {
+ CrawlRequest,
+ type ScrapeOptions,
+ crawlRequestSchema,
+ scrapeRequestSchema,
+ toV0CrawlerOptions,
+} from "../../controllers/v2/types";
+import { createWebhookSender, WebhookEvent } from "../webhook";
+import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
+import {
+ getMonitorForUpdate,
+ getMonitorPage,
+ countMonitorCheckPages,
+ hashMonitorUrl,
+ insertMonitorCheckPages,
+ listActiveMonitorPages,
+ listMonitorCheckPages,
+ listRunningMonitorChecks,
+ markMonitorRunning,
+ updateMonitorCheck,
+ updateMonitorScheduleAfterRun,
+ upsertMonitorPage,
+} from "./store";
+import type {
+ MonitorCheckPageInsert,
+ MonitorCheckRow,
+ MonitorRow,
+ MonitorTarget,
+} from "./types";
+import { withMarkdownFormat } from "./types";
+import { redisEvictConnection } from "../redis";
+import type { MonitorCheckJobData } from "./queue";
+
+const logger = _logger.child({ module: "monitoring-runner" });
+const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
+type PageResult = MonitorCheckPageInsert & {
+ emailStatus?: string;
+};
+
+function withMonitorScrapeDefaults(
+ options: Record<string, unknown>,
+): ScrapeOptions {
+ return {
+ maxAge: 0,
+ ...withMarkdownFormat(options),
+ };
+}
+
+function getDocumentUrl(doc: any, fallback: string): string {
+ return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;
+}
+
+function getDocumentStatusCode(doc: any): number | null {
+ return typeof doc?.metadata?.statusCode === "number"
+ ? doc.metadata.statusCode
+ : null;
+}
+
+function estimateActualCredits(doc: any, options: any): number {
+ if (typeof doc?.metadata?.creditsUsed === "number") {
+ return doc.metadata.creditsUsed;
+ }
+ const formats = Array.isArray(options?.formats) ? options.formats : [];
+ const hasJson = formats.some((format: any) =>
+ typeof format === "string" ? format === "json" : format?.type === "json",
+ );
+ return hasJson ? 5 : 1;
+}
+
+async function runSingleScrape(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ target: MonitorTarget;
+ url: string;
+ requestId?: string;
+}): Promise<{ scrapeId: string; doc: any; credits: number }> {
+ const scrapeId = uuidv7();
+ const scrapeOptions = scrapeRequestSchema.parse({
+ url: params.url,
+ ...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
+ origin: "monitor",
+ });
+
+ await logRequest({
+ id: scrapeId,
+ kind: "scrape",
+ api_version: "v2",
+ team_id: params.monitor.team_id,
+ origin: "monitor",
+ integration: null,
+ target_hint: params.url,
+ zeroDataRetention: false,
+ api_key_id: null,
+ });
+
+ const job: NuQJob<ScrapeJobData> = {
+ id: scrapeId,
+ status: "active",
+ createdAt: new Date(),
+ priority: 20,
+ data: {
+ mode: "single_urls",
+ url: params.url,
+ team_id: params.monitor.team_id,
+ scrapeOptions,
+ internalOptions: {
+ teamId: params.monitor.team_id,
+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
+ bypassBilling: true,
+ zeroDataRetention: false,
+ },
+ skipNuq: true,
+ origin: "monitor",
+ integration: null,
+ billing: { endpoint: "monitor", jobId: params.check.id },
+ requestId: params.requestId,
+ zeroDataRetention: false,
+ apiKeyId: null,
+ },
+ };
+
+ const doc = await processJobInternal(job);
+ return {
+ scrapeId,
+ doc,
+ credits: estimateActualCredits(doc, scrapeOptions),
+ };
+}
+
+async function diffAndPersistPage(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ target: MonitorTarget;
+ url: string;
+ scrapeId: string;
+ doc: any;
+ source: "explicit" | "discovered";
+}): Promise<PageResult> {
+ const previous = await getMonitorPage({
+ monitorId: params.monitor.id,
+ targetId: params.target.id,
+ url: params.url,
+ });
+
+ let status: PageResult["status"] = "new";
+ let diffGcsKey: string | null = null;
+ let diffTextBytes: number | null = null;
+ let diffJsonBytes: number | null = null;
+
+ if (previous?.last_scrape_id && !previous.is_removed) {
+ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];
+ const previousMarkdown = previousDoc?.markdown;
+ const currentMarkdown = params.doc?.markdown;
+
+ if (previousMarkdown && currentMarkdown) {
+ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);
+ status = diff.status;
+
+ if (diff.status === "changed") {
+ diffGcsKey = monitorDiffGcsKey({
+ teamId: params.monitor.team_id,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ pageId: uuidv7(),
+ });
+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, {
+ url: params.url,
+ previousScrapeId: previous.last_scrape_id,
+ currentScrapeId: params.scrapeId,
+ text: diff.text,
+ json: diff.json,
+ generatedAt: new Date().toISOString(),
+ });
+ diffTextBytes = sizes.textBytes;
+ diffJsonBytes = sizes.jsonBytes;
+ }
+ } else {
+ status = "changed";
+ }
+ }
+
+ await upsertMonitorPage({
+ monitorId: params.monitor.id,
+ teamId: params.monitor.team_id,
+ targetId: params.target.id,
+ url: params.url,
+ source: params.source,
+ checkId: params.check.id,
+ scrapeId: params.scrapeId,
+ status,
+ metadata: {
+ title: params.doc?.metadata?.title ?? null,
+ statusCode: getDocumentStatusCode(params.doc),
+ },
+ });
+
+ return {
+ check_id: params.check.id,
+ monitor_id: params.monitor.id,
+ team_id: params.monitor.team_id,
+ target_id: params.target.id,
+ url: params.url,
+ url_hash: hashMonitorUrl(params.url),
+ status,
+ previous_scrape_id: previous?.last_scrape_id ?? null,
+ current_scrape_id: params.scrapeId,
+ diff_gcs_key: diffGcsKey,
+ diff_text_bytes: diffTextBytes,
+ diff_json_bytes: diffJsonBytes,
+ status_code: getDocumentStatusCode(params.doc),
+ metadata: {
+ title: params.doc?.metadata?.title ?? null,
+ },
+ emailStatus: status,
+ };
+}
+
+async function runScrapeTarget(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ target: MonitorTarget;
+}): Promise<{ pages: PageResult[]; credits: number; targetResult: any }> {
+ if (params.target.type !== "scrape") {
+ return { pages: [], credits: 0, targetResult: null };
+ }
+
+ const pages: PageResult[] = [];
+ let credits = 0;
+
+ for (const url of params.target.urls) {
+ try {
+ const result = await runSingleScrape({
+ monitor: params.monitor,
+ check: params.check,
+ target: params.target,
+ url,
+ });
+ credits += result.credits;
+ pages.push(
+ await diffAndPersistPage({
+ monitor: params.monitor,
+ check: params.check,
+ target: params.target,
+ url,
+ scrapeId: result.scrapeId,
+ doc: result.doc,
+ source: "explicit",
+ }),
+ );
+ } catch (error) {
+ pages.push({
+ check_id: params.check.id,
+ monitor_id: params.monitor.id,
+ team_id: params.monitor.team_id,
+ target_id: params.target.id,
+ url,
+ url_hash: hashMonitorUrl(url),
+ status: "error",
+ error: error instanceof Error ? error.message : String(error),
+ emailStatus: "error",
+ });
+ }
+ }
+
+ return {
+ pages,
+ credits,
+ targetResult: {
+ targetId: params.target.id,
+ type: params.target.type,
+ pages: pages.length,
+ credits,
+ },
+ };
+}
+
+async function runCrawlTarget(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ target: MonitorTarget;
+}): Promise<{ pages: PageResult[]; credits: number; targetResult: any }> {
+ if (params.target.type !== "crawl") {
+ return { pages: [], credits: 0, targetResult: null };
+ }
+
+ const crawlId = uuidv7();
+ const body = crawlRequestSchema.parse({
+ url: params.target.url,
+ ...(params.target.crawlOptions ?? {}),
+ scrapeOptions: withMarkdownFormat(params.target.scrapeOptions ?? {}),
+ origin: "monitor",
+ }) as CrawlRequest;
+
+ await logRequest({
+ id: crawlId,
+ kind: "crawl",
+ api_version: "v2",
+ team_id: params.monitor.team_id,
+ origin: "monitor",
+ integration: null,
+ target_hint: body.url,
+ zeroDataRetention: false,
+ api_key_id: null,
+ });
+
+ const crawlerOptions = {
+ ...body,
+ url: undefined,
+ scrapeOptions: undefined,
+ prompt: undefined,
+ };
+
+ const sc: StoredCrawl = {
+ originUrl: body.url,
+ crawlerOptions: toV0CrawlerOptions(crawlerOptions),
+ scrapeOptions: body.scrapeOptions,
+ internalOptions: {
+ disableSmartWaitCache: true,
+ teamId: params.monitor.team_id,
+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
+ zeroDataRetention: false,
+ bypassBilling: true,
+ },
+ team_id: params.monitor.team_id,
+ createdAt: Date.now(),
+ maxConcurrency: body.maxConcurrency,
+ zeroDataRetention: false,
+ };
+
+ const crawler = crawlToCrawler(crawlId, sc, null);
+ try {
+ sc.robots = await crawler.getRobotsTxt(
+ body.scrapeOptions.skipTlsVerification,
+ );
+ } catch {
+ // Crawls tolerate robots fetch failures in the public controller too.
+ }
+
+ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);
+ await saveCrawl(crawlId, sc);
+ await markCrawlActive(crawlId);
+
+ await _addScrapeJobToBullMQ(
+ {
+ url: body.url,
+ mode: "kickoff",
+ team_id: params.monitor.team_id,
+ crawlerOptions,
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: "monitor",
+ integration: null,
+ billing: { endpoint: "monitor", jobId: params.check.id },
+ crawl_id: crawlId,
+ v1: true,
+ zeroDataRetention: false,
+ apiKeyId: null,
+ },
+ uuidv7(),
+ );
+
+ const started = Date.now();
+ let status = "scraping";
+ let total = 0;
+ while (Date.now() - started < 30 * 60 * 1000) {
+ const group = await crawlGroup.getGroup(crawlId);
+ const stats = await scrapeQueue.getGroupNumericStats(crawlId, logger);
+ status = group?.status ?? "scraping";
+ total =
+ (stats.completed ?? 0) +
+ (stats.active ?? 0) +
+ (stats.queued ?? 0) +
+ (stats.backlog ?? 0);
+ if (status !== "active" && status !== "scraping") break;
+ await poll(1000);
+ }
+
+ const doneJobs = await scrapeQueue.getCrawlJobsForListing(
+ crawlId,
+ Math.max(total, 1),
+ 0,
+ logger,
+ );
+
+ const pages: PageResult[] = [];
+ const seen = new Set<string>();
+ let credits = 0;
+
+ for (const job of doneJobs) {
+ const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];
+ if (!doc) continue;
+ const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);
+ seen.add(hashMonitorUrl(url));
+ credits += estimateActualCredits(doc, body.scrapeOptions);
+ pages.push(
+ await diffAndPersistPage({
+ monitor: params.monitor,
+ check: params.check,
+ target: params.target,
+ url,
+ scrapeId: job.id,
+ doc,
+ source: "discovered",
+ }),
+ );
+ }
+
+ if (status === "completed") {
+ const previousPages = await listActiveMonitorPages({
+ monitorId: params.monitor.id,
+ targetId: params.target.id,
+ });
+ for (const previous of previousPages) {
+ if (seen.has(previous.url_hash)) continue;
+ await upsertMonitorPage({
+ monitorId: params.monitor.id,
+ teamId: params.monitor.team_id,
+ targetId: params.target.id,
+ url: previous.url,
+ source: previous.source,
+ checkId: params.check.id,
+ scrapeId: previous.last_scrape_id,
+ status: "removed",
+ metadata: previous.metadata,
+ });
+ pages.push({
+ check_id: params.check.id,
+ monitor_id: params.monitor.id,
+ team_id: params.monitor.team_id,
+ target_id: params.target.id,
+ url: previous.url,
+ url_hash: previous.url_hash,
+ status: "removed",
+ previous_scrape_id: previous.last_scrape_id,
+ current_scrape_id: null,
+ emailStatus: "removed",
+ });
+ }
+ }
+
+ return {
+ pages,
+ credits,
+ targetResult: {
+ targetId: params.target.id,
+ type: params.target.type,
+ crawlId,
+ status,
+ pages: pages.length,
+ credits,
+ },
+ };
+}
+
+function summarize(pages: PageResult[]) {
+ return {
+ totalPages: pages.length,
+ same: pages.filter(page => page.status === "same").length,
+ changed: pages.filter(page => page.status === "changed").length,
+ new: pages.filter(page => page.status === "new").length,
+ removed: pages.filter(page => page.status === "removed").length,
+ error: pages.filter(page => page.status === "error").length,
+ };
+}
+
+async function billMonitorCheck(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ actualCredits: number;
+ lockId: string | null;
+}): Promise<void> {
+ if (params.lockId) {
+ await autumnService.finalizeCreditsLock({
+ lockId: params.lockId,
+ action: "confirm",
+ overrideValue: params.actualCredits,
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: params.check.id,
+ },
+ });
+ }
+
+ if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION) return;
+
+ await getBillingQueue().add(
+ "bill_team",
+ {
+ team_id: params.monitor.team_id,
+ subscription_id: undefined,
+ credits: params.actualCredits,
+ billing: { endpoint: "monitor", jobId: params.check.id },
+ is_extract: false,
+ timestamp: new Date().toISOString(),
+ originating_job_id: params.check.id,
+ api_key_id: null,
+ autumnTrackInRequest: Boolean(params.lockId),
+ },
+ {
+ jobId: uuidv7(),
+ priority: 10,
+ },
+ );
+}
+
+async function sendNotifications(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ pages: PageResult[];
+}): Promise<{ webhook?: unknown; email?: unknown }> {
+ const payload = {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ status: params.check.status,
+ summary: toSummaryObject(params.check),
+ };
+
+ let webhookStatus: unknown = { attempted: false };
+ if (params.monitor.webhook) {
+ const sender = await createWebhookSender({
+ teamId: params.monitor.team_id,
+ jobId: params.check.id,
+ webhook: params.monitor.webhook as any,
+ v0: false,
+ });
+ await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {
+ success: params.check.status === "completed",
+ data: payload,
+ error: params.check.error ?? undefined,
+ });
+ webhookStatus = { attempted: true, success: true };
+ }
+
+ const emailStatus = await sendMonitoringEmailSummary({
+ monitor: params.monitor,
+ check: params.check,
+ pages: params.pages
+ .filter(page => page.status !== "same")
+ .slice(0, 25)
+ .map(page => ({
+ url: page.url,
+ status: page.status,
+ error: page.error,
+ })),
+ });
+
+ return {
+ webhook: webhookStatus,
+ email: emailStatus,
+ };
+}
+
+async function enqueueMonitorScrapeTarget(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ target: MonitorTarget;
+}): Promise<{ targetId: string; type: "scrape"; expectedJobs: string[] }> {
+ if (params.target.type !== "scrape") {
+ throw new Error("Expected scrape target");
+ }
+
+ const expectedJobs: string[] = [];
+ for (const url of params.target.urls) {
+ const scrapeId = uuidv7();
+ const scrapeOptions = scrapeRequestSchema.parse({
+ url,
+ ...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
+ origin: "monitor",
+ });
+
+ await logRequest({
+ id: scrapeId,
+ kind: "scrape",
+ api_version: "v2",
+ team_id: params.monitor.team_id,
+ origin: "monitor",
+ integration: null,
+ target_hint: url,
+ zeroDataRetention: false,
+ api_key_id: null,
+ });
+
+ await addScrapeJob(
+ {
+ mode: "single_urls",
+ url,
+ team_id: params.monitor.team_id,
+ scrapeOptions,
+ internalOptions: {
+ teamId: params.monitor.team_id,
+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
+ bypassBilling: true,
+ zeroDataRetention: false,
+ },
+ origin: "monitor",
+ integration: null,
+ billing: { endpoint: "monitor", jobId: params.check.id },
+ zeroDataRetention: false,
+ apiKeyId: null,
+ monitoring: {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ targetId: params.target.id,
+ source: "explicit",
+ },
+ },
+ scrapeId,
+ 20,
+ );
+ expectedJobs.push(scrapeId);
+ }
+
+ return { targetId: params.target.id, type: "scrape", expectedJobs };
+}
+
+async function enqueueMonitorCrawlTarget(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ target: MonitorTarget;
+}): Promise<{ targetId: string; type: "crawl"; crawlId: string }> {
+ if (params.target.type !== "crawl") {
+ throw new Error("Expected crawl target");
+ }
+
+ const crawlId = uuidv7();
+ const body = crawlRequestSchema.parse({
+ url: params.target.url,
+ ...(params.target.crawlOptions ?? {}),
+ scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
+ origin: "monitor",
+ }) as CrawlRequest;
+
+ await logRequest({
+ id: crawlId,
+ kind: "crawl",
+ api_version: "v2",
+ team_id: params.monitor.team_id,
+ origin: "monitor",
+ integration: null,
+ target_hint: body.url,
+ zeroDataRetention: false,
+ api_key_id: null,
+ });
+
+ const crawlerOptions = {
+ ...body,
+ url: undefined,
+ scrapeOptions: undefined,
+ prompt: undefined,
+ };
+
+ const sc: StoredCrawl = {
+ originUrl: body.url,
+ crawlerOptions: toV0CrawlerOptions(crawlerOptions),
+ scrapeOptions: body.scrapeOptions,
+ internalOptions: {
+ disableSmartWaitCache: true,
+ teamId: params.monitor.team_id,
+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
+ zeroDataRetention: false,
+ bypassBilling: true,
+ },
+ team_id: params.monitor.team_id,
+ createdAt: Date.now(),
+ maxConcurrency: body.maxConcurrency,
+ zeroDataRetention: false,
+ };
+
+ const crawler = crawlToCrawler(crawlId, sc, null);
+ try {
+ sc.robots = await crawler.getRobotsTxt(
+ body.scrapeOptions.skipTlsVerification,
+ );
+ } catch {
+ // Non-fatal, same as the public crawl controller.
+ }
+
+ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);
+ await saveCrawl(crawlId, sc);
+ await markCrawlActive(crawlId);
+
+ await _addScrapeJobToBullMQ(
+ {
+ url: body.url,
+ mode: "kickoff",
+ team_id: params.monitor.team_id,
+ crawlerOptions,
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: "monitor",
+ integration: null,
+ billing: { endpoint: "monitor", jobId: params.check.id },
+ crawl_id: crawlId,
+ v1: true,
+ zeroDataRetention: false,
+ apiKeyId: null,
+ monitoring: {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ targetId: params.target.id,
+ source: "discovered",
+ },
+ },
+ uuidv7(),
+ );
+
+ return { targetId: params.target.id, type: "crawl", crawlId };
+}
+
+export async function processMonitorCheckJob(
+ job: MonitorCheckJobData,
+): Promise<void> {
+ const monitor = await getMonitorForUpdate(job.teamId, job.monitorId);
+ if (!monitor) {
+ throw new Error("Monitor not found");
+ }
+
+ await markMonitorRunning({
+ monitorId: monitor.id,
+ checkId: job.checkId,
+ });
+
+ let check = await updateMonitorCheck(job.checkId, {
+ status: "running",
+ started_at: new Date().toISOString(),
+ });
+
+ let lockId: string | null = null;
+ try {
+ lockId = await autumnService.lockCredits({
+ teamId: monitor.team_id,
+ value: check.estimated_credits ?? 1,
+ lockId: `monitor_${check.id}`,
+ expiresAt: Date.now() + 60 * 60 * 1000,
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: check.id,
+ },
+ });
+
+ check = await updateMonitorCheck(check.id, {
+ autumn_lock_id: lockId,
+ reserved_credits: lockId ? (check.estimated_credits ?? 1) : null,
+ billing_status: lockId ? "reserved" : "not_applicable",
+ });
+
+ const targetResults: unknown[] = [];
+
+ for (const target of monitor.targets) {
+ const result =
+ target.type === "scrape"
+ ? await enqueueMonitorScrapeTarget({ monitor, check, target })
+ : await enqueueMonitorCrawlTarget({ monitor, check, target });
+ targetResults.push(result);
+ }
+
+ await updateMonitorCheck(check.id, {
+ target_results: targetResults,
+ });
+ } catch (error) {
+ if (lockId) {
+ await autumnService.finalizeCreditsLock({
+ lockId,
+ action: "release",
+ properties: {
+ source: "monitorCheck",
+ endpoint: "monitor",
+ jobId: check.id,
+ },
+ });
+ }
+
+ check = await updateMonitorCheck(check.id, {
+ status: "failed",
+ finished_at: new Date().toISOString(),
+ billing_status: lockId ? "released" : "failed",
+ error: error instanceof Error ? error.message : String(error),
+ });
+
+ await sendNotifications({
+ monitor,
+ check,
+ pages: [],
+ }).catch(err =>
+ logger.warn("Failed to send monitor failure notifications", {
+ error: err,
+ }),
+ );
+
+ await updateMonitorScheduleAfterRun({
+ monitor,
+ check,
+ });
+
+ throw error;
+ }
+}
+
+async function processRemovedPagesForCompletedCrawls(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ targetResults: any[];
+}): Promise<void> {
+ for (const target of params.targetResults) {
+ if (target?.type !== "crawl" || target.removedProcessed) continue;
+
+ const group = await crawlGroup.getGroup(target.crawlId);
+ if (group?.status !== "completed") continue;
+
+ const checkPages = await listMonitorCheckPages({
+ teamId: params.monitor.team_id,
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ limit: 100000,
+ skip: 0,
+ });
+ const seen = new Set(
+ checkPages
+ .filter(page => page.target_id === target.targetId)
+ .map(page => page.url_hash),
+ );
+ const activePages = await listActiveMonitorPages({
+ monitorId: params.monitor.id,
+ targetId: target.targetId,
+ });
+
+ const removed: MonitorCheckPageInsert[] = [];
+ for (const previous of activePages) {
+ if (seen.has(previous.url_hash)) continue;
+ await upsertMonitorPage({
+ monitorId: params.monitor.id,
+ teamId: params.monitor.team_id,
+ targetId: target.targetId,
+ url: previous.url,
+ source: previous.source,
+ checkId: params.check.id,
+ scrapeId: previous.last_scrape_id,
+ status: "removed",
+ metadata: previous.metadata,
+ });
+ removed.push({
+ check_id: params.check.id,
+ monitor_id: params.monitor.id,
+ team_id: params.monitor.team_id,
+ target_id: target.targetId,
+ url: previous.url,
+ url_hash: previous.url_hash,
+ status: "removed" as const,
+ previous_scrape_id: previous.last_scrape_id,
+ current_scrape_id: null,
+ });
+ }
+
+ await insertMonitorCheckPages(removed);
+ target.removedProcessed = true;
+ }
+}
+
+async function isMonitorCheckComplete(
+ check: MonitorCheckRow,
+): Promise<boolean> {
+ const targetResults = Array.isArray(check.target_results)
+ ? (check.target_results as any[])
+ : [];
+
+ if (targetResults.length === 0) return false;
+
+ for (const target of targetResults) {
+ if (target?.type === "scrape") {
+ const expected = Array.isArray(target.expectedJobs)
+ ? target.expectedJobs.length
+ : 0;
+ const recorded = await countMonitorCheckPages({
+ checkId: check.id,
+ targetId: target.targetId,
+ });
+ if (recorded < expected) return false;
+ } else if (target?.type === "crawl") {
+ const group = await crawlGroup.getGroup(target.crawlId);
+ if (!group || group.status === "active") return false;
+
+ const stats = await scrapeQueue.getGroupNumericStats(
+ target.crawlId,
+ logger,
+ );
+ const unfinished =
+ (stats.active ?? 0) + (stats.queued ?? 0) + (stats.backlog ?? 0);
+ if (unfinished > 0) return false;
+ }
+ }
+
+ return true;
+}
+
+export async function reconcileRunningMonitorChecks(
+ limit: number = 50,
+): Promise<void> {
+ const checks = await listRunningMonitorChecks(limit);
+ for (const check of checks) {
+ const lockKey = `monitor-check-finalize:${check.id}`;
+ const lock = await redisEvictConnection.set(lockKey, "1", "EX", 60, "NX");
+ if (lock !== "OK") continue;
+
+ try {
+ const monitor = await getMonitorForUpdate(
+ check.team_id,
+ check.monitor_id,
+ );
+ if (!monitor) continue;
+
+ const targetResults = Array.isArray(check.target_results)
+ ? ([...check.target_results] as any[])
+ : [];
+
+ await processRemovedPagesForCompletedCrawls({
+ monitor,
+ check,
+ targetResults,
+ });
+
+ if (
+ !(await isMonitorCheckComplete({
+ ...check,
+ target_results: targetResults,
+ }))
+ ) {
+ await updateMonitorCheck(check.id, { target_results: targetResults });
+ continue;
+ }
+
+ const [same, changed, newCount, removed, errorCount] = await Promise.all([
+ countMonitorCheckPages({ checkId: check.id, status: "same" }),
+ countMonitorCheckPages({ checkId: check.id, status: "changed" }),
+ countMonitorCheckPages({ checkId: check.id, status: "new" }),
+ countMonitorCheckPages({ checkId: check.id, status: "removed" }),
+ countMonitorCheckPages({ checkId: check.id, status: "error" }),
+ ]);
+ const totalPages = same + changed + newCount + removed + errorCount;
+ const actualCredits = totalPages;
+
+ let finalized = await updateMonitorCheck(check.id, {
+ status: errorCount > 0 ? "partial" : "completed",
+ finished_at: new Date().toISOString(),
+ actual_credits: actualCredits,
+ billing_status: check.autumn_lock_id ? "confirmed" : "not_applicable",
+ total_pages: totalPages,
+ same_count: same,
+ changed_count: changed,
+ new_count: newCount,
+ removed_count: removed,
+ error_count: errorCount,
+ target_results: targetResults,
+ });
+
+ try {
+ await billMonitorCheck({
+ monitor,
+ check: finalized,
+ actualCredits,
+ lockId: check.autumn_lock_id,
+ });
+ } catch (error) {
+ logger.warn("Failed to bill monitor check during reconciliation", {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ error,
+ });
+ finalized = await updateMonitorCheck(check.id, {
+ billing_status: "failed",
+ }).catch(updateError => {
+ logger.warn("Failed to record monitor check billing failure", {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ error: updateError,
+ });
+ return finalized;
+ });
+ }
+
+ let notificationStatus: { webhook?: unknown; email?: unknown } | null =
+ null;
+ try {
+ const pages = (await listMonitorCheckPages({
+ teamId: monitor.team_id,
+ monitorId: monitor.id,
+ checkId: check.id,
+ limit: 100,
+ skip: 0,
+ })) as PageResult[];
+
+ notificationStatus = await sendNotifications({
+ monitor,
+ check: finalized,
+ pages,
+ });
+
+ finalized = await updateMonitorCheck(check.id, {
+ notification_status: notificationStatus,
+ webhook_payload: notificationStatus.webhook
+ ? { summary: toSummaryObject(finalized) }
+ : null,
+ email_payload: notificationStatus.email
+ ? { summary: toSummaryObject(finalized) }
+ : null,
+ });
+ } catch (error) {
+ logger.warn("Failed to send monitor check notifications", {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ error,
+ });
+ notificationStatus = {
+ webhook: {
+ attempted: !!monitor.webhook,
+ success: false,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ email: {
+ attempted: !!monitor.notification?.email?.enabled,
+ success: false,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ };
+ finalized = await updateMonitorCheck(check.id, {
+ notification_status: notificationStatus,
+ }).catch(updateError => {
+ logger.warn("Failed to record monitor check notification failure", {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ error: updateError,
+ });
+ return finalized;
+ });
+ }
+
+ await updateMonitorScheduleAfterRun({
+ monitor,
+ check: finalized,
+ summary: toSummaryObject(finalized),
+ });
+
+ logger.info("Reconciled monitor check", {
+ monitorId: monitor.id,
+ checkId: finalized.id,
+ status: finalized.status,
+ totalPages,
+ same,
+ changed,
+ new: newCount,
+ removed,
+ errors: errorCount,
+ });
+ } catch (error) {
+ logger.warn("Failed to reconcile monitor check", {
+ error,
+ checkId: check.id,
+ });
+ } finally {
+ await redisEvictConnection.del(lockKey);
+ }
+ }
+}
+
+function toSummaryObject(check: MonitorCheckRow) {
+ return {
+ totalPages: check.total_pages,
+ same: check.same_count,
+ changed: check.changed_count,
+ new: check.new_count,
+ removed: check.removed_count,
+ error: check.error_count,
+ };
+}
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
new file mode 100644
index 000000000..99b817b34
--- /dev/null
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -0,0 +1,75 @@
+import { randomUUID } from "crypto";
+import { logger as _logger } from "../../lib/logger";
+import { addMonitorCheckJob } from "./queue";
+import {
+ advanceMonitorAfterSkippedCheck,
+ claimDueMonitors,
+ createMonitorCheck,
+ updateMonitorCheck,
+} from "./store";
+
+const logger = _logger.child({ module: "monitoring-scheduler" });
+
+export async function enqueueMonitorCheck(params: {
+ monitorId: string;
+ checkId: string;
+ teamId: string;
+}): Promise<void> {
+ await addMonitorCheckJob(params);
+}
+
+export async function enqueueDueMonitorChecks(
+ params: {
+ workerId?: string;
+ limit?: number;
+ leaseSeconds?: number;
+ } = {},
+): Promise<number> {
+ const workerId = params.workerId ?? `monitor-scheduler-${randomUUID()}`;
+ const monitors = await claimDueMonitors({
+ workerId,
+ limit: params.limit ?? 10,
+ leaseSeconds: params.leaseSeconds ?? 60,
+ });
+
+ let enqueued = 0;
+ for (const monitor of monitors) {
+ try {
+ if (monitor.current_check_id) {
+ const skipped = await createMonitorCheck({
+ monitor,
+ trigger: "scheduled",
+ scheduledFor: monitor.next_run_at,
+ status: "skipped_overlap",
+ });
+ const finished = await updateMonitorCheck(skipped.id, {
+ status: "skipped_overlap",
+ finished_at: new Date().toISOString(),
+ error: "Previous monitor check is still running.",
+ });
+ await advanceMonitorAfterSkippedCheck({ monitor, check: finished });
+ continue;
+ }
+
+ const check = await createMonitorCheck({
+ monitor,
+ trigger: "scheduled",
+ scheduledFor: monitor.next_run_at,
+ });
+ await enqueueMonitorCheck({
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ });
+ enqueued++;
+ } catch (error) {
+ logger.error("Failed to enqueue due monitor check", {
+ error,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ }
+ }
+
+ return enqueued;
+}
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
new file mode 100644
index 000000000..2d4e51dff
--- /dev/null
+++ b/apps/api/src/services/monitoring/store.ts
@@ -0,0 +1,589 @@
+import { createHash } from "crypto";
+import { v7 as uuidv7 } from "uuid";
+import { supabase_rr_service, supabase_service } from "../supabase";
+import { getNextMonitorRunAt, estimateRunsPerMonth } from "./cron";
+import type {
+ CreateMonitorRequest,
+ MonitorCheckPageInsert,
+ MonitorCheckRow,
+ MonitorPageRow,
+ MonitorRow,
+ MonitorSummary,
+ MonitorTarget,
+ UpdateMonitorRequest,
+} from "./types";
+
+export function hashMonitorUrl(url: string): string {
+ return `\\x${createHash("sha256").update(url).digest("hex")}`;
+}
+
+function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {
+ return targets.map(target => ({
+ ...target,
+ id: typeof target.id === "string" ? target.id : uuidv7(),
+ })) as MonitorTarget[];
+}
+
+function estimateTargetCredits(target: MonitorTarget): number {
+ if (target.type === "scrape") {
+ return target.urls.length;
+ }
+
+ const limit =
+ typeof target.crawlOptions?.limit === "number"
+ ? target.crawlOptions.limit
+ : 10000;
+ return Math.max(1, limit);
+}
+
+export function estimateMonitorCreditsPerRun(targets: MonitorTarget[]): number {
+ return targets.reduce(
+ (sum, target) => sum + estimateTargetCredits(target),
+ 0,
+ );
+}
+
+function toMonitorSummary(check: MonitorCheckRow): MonitorSummary {
+ return {
+ totalPages: check.total_pages,
+ same: check.same_count,
+ changed: check.changed_count,
+ new: check.new_count,
+ removed: check.removed_count,
+ error: check.error_count,
+ };
+}
+
+function throwIfError(error: any, message: string): void {
+ if (error) {
+ throw new Error(`${message}: ${error.message ?? JSON.stringify(error)}`);
+ }
+}
+
+export async function createMonitor(params: {
+ teamId: string;
+ input: CreateMonitorRequest;
+ nextRunAt: Date;
+ intervalMs: number;
+}): Promise<MonitorRow> {
+ const targets = ensureTargetIds(params.input.targets);
+ const estimatedCreditsPerRun = estimateMonitorCreditsPerRun(targets);
+ const estimatedCreditsPerMonth =
+ estimatedCreditsPerRun * estimateRunsPerMonth(params.intervalMs);
+
+ const { data, error } = await supabase_service
+ .from("monitors")
+ .insert({
+ id: uuidv7(),
+ team_id: params.teamId,
+ name: params.input.name,
+ schedule_cron: params.input.schedule.cron,
+ schedule_timezone: params.input.schedule.timezone,
+ next_run_at: params.nextRunAt.toISOString(),
+ retention_days: params.input.retentionDays,
+ estimated_credits_per_month: estimatedCreditsPerMonth,
+ targets,
+ webhook: params.input.webhook ?? null,
+ notification: params.input.notification ?? null,
+ })
+ .select("*")
+ .single();
+
+ throwIfError(error, "Failed to create monitor");
+ return data as MonitorRow;
+}
+
+export async function listMonitors(params: {
+ teamId: string;
+ limit: number;
+ offset: number;
+}): Promise<MonitorRow[]> {
+ const { data, error } = await supabase_rr_service
+ .from("monitors")
+ .select("*")
+ .eq("team_id", params.teamId)
+ .neq("status", "deleted")
+ .order("created_at", { ascending: false })
+ .range(params.offset, params.offset + params.limit - 1);
+
+ throwIfError(error, "Failed to list monitors");
+ return (data ?? []) as MonitorRow[];
+}
+
+export async function getMonitor(
+ teamId: string,
+ monitorId: string,
+): Promise<MonitorRow | null> {
+ const { data, error } = await supabase_rr_service
+ .from("monitors")
+ .select("*")
+ .eq("id", monitorId)
+ .eq("team_id", teamId)
+ .neq("status", "deleted")
+ .maybeSingle();
+
+ throwIfError(error, "Failed to get monitor");
+ return data as MonitorRow | null;
+}
+
+export async function getMonitorForUpdate(
+ teamId: string,
+ monitorId: string,
+): Promise<MonitorRow | null> {
+ const { data, error } = await supabase_service
+ .from("monitors")
+ .select("*")
+ .eq("id", monitorId)
+ .eq("team_id", teamId)
+ .neq("status", "deleted")
+ .maybeSingle();
+
+ throwIfError(error, "Failed to get monitor");
+ return data as MonitorRow | null;
+}
+
+export async function updateMonitor(params: {
+ teamId: string;
+ monitorId: string;
+ input: UpdateMonitorRequest;
+ nextRunAt?: Date;
+ intervalMs?: number;
+}): Promise<MonitorRow | null> {
+ const patch: Record<string, unknown> = {
+ updated_at: new Date().toISOString(),
+ };
+
+ if (params.input.name !== undefined) patch.name = params.input.name;
+ if (params.input.status !== undefined) patch.status = params.input.status;
+ if (params.input.webhook !== undefined)
+ patch.webhook = params.input.webhook ?? null;
+ if (params.input.notification !== undefined) {
+ patch.notification = params.input.notification ?? null;
+ }
+ if (params.input.retentionDays !== undefined) {
+ patch.retention_days = params.input.retentionDays;
+ }
+ if (params.input.targets !== undefined) {
+ const targets = ensureTargetIds(params.input.targets);
+ patch.targets = targets;
+ if (params.intervalMs !== undefined) {
+ patch.estimated_credits_per_month =
+ estimateMonitorCreditsPerRun(targets) *
+ estimateRunsPerMonth(params.intervalMs);
+ }
+ }
+ if (params.input.schedule !== undefined) {
+ patch.schedule_cron = params.input.schedule.cron;
+ patch.schedule_timezone = params.input.schedule.timezone;
+ patch.next_run_at = params.nextRunAt?.toISOString() ?? null;
+ }
+
+ const { data, error } = await supabase_service
+ .from("monitors")
+ .update(patch)
+ .eq("id", params.monitorId)
+ .eq("team_id", params.teamId)
+ .neq("status", "deleted")
+ .select("*")
+ .maybeSingle();
+
+ throwIfError(error, "Failed to update monitor");
+ return data as MonitorRow | null;
+}
+
+export async function deleteMonitor(params: {
+ teamId: string;
+ monitorId: string;
+}): Promise<boolean> {
+ const { data, error } = await supabase_service
+ .from("monitors")
+ .update({
+ status: "deleted",
+ deleted_at: new Date().toISOString(),
+ next_run_at: null,
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", params.monitorId)
+ .eq("team_id", params.teamId)
+ .neq("status", "deleted")
+ .select("id")
+ .maybeSingle();
+
+ throwIfError(error, "Failed to delete monitor");
+ return !!data;
+}
+
+export async function createMonitorCheck(params: {
+ monitor: MonitorRow;
+ trigger: "scheduled" | "manual";
+ scheduledFor?: string | null;
+ status?: MonitorCheckRow["status"];
+}): Promise<MonitorCheckRow> {
+ const estimated = estimateMonitorCreditsPerRun(params.monitor.targets);
+ const { data, error } = await supabase_service
+ .from("monitor_checks")
+ .insert({
+ id: uuidv7(),
+ monitor_id: params.monitor.id,
+ team_id: params.monitor.team_id,
+ trigger: params.trigger,
+ status: params.status ?? "queued",
+ scheduled_for: params.scheduledFor ?? null,
+ estimated_credits: estimated,
+ })
+ .select("*")
+ .single();
+
+ throwIfError(error, "Failed to create monitor check");
+ return data as MonitorCheckRow;
+}
+
+export async function markMonitorRunning(params: {
+ monitorId: string;
+ checkId: string;
+}): Promise<void> {
+ const { error } = await supabase_service
+ .from("monitors")
+ .update({
+ current_check_id: params.checkId,
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", params.monitorId)
+ .is("current_check_id", null);
+
+ throwIfError(error, "Failed to mark monitor running");
+}
+
+export async function updateMonitorScheduleAfterRun(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ summary?: MonitorSummary;
+}): Promise<void> {
+ const nextRunAt =
+ params.monitor.status === "active"
+ ? getNextMonitorRunAt(
+ params.monitor.schedule_cron,
+ new Date(),
+ params.monitor.schedule_timezone,
+ ).toISOString()
+ : null;
+ const { error } = await supabase_service
+ .from("monitors")
+ .update({
+ current_check_id: null,
+ locked_at: null,
+ locked_until: null,
+ last_run_at: params.check.finished_at ?? new Date().toISOString(),
+ last_check_id: params.check.id,
+ next_run_at: nextRunAt,
+ last_check_summary: params.summary ?? toMonitorSummary(params.check),
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", params.monitor.id);
+
+ throwIfError(error, "Failed to update monitor after run");
+}
+
+export async function advanceMonitorAfterSkippedCheck(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+}): Promise<void> {
+ const nextRunAt =
+ params.monitor.status === "active"
+ ? getNextMonitorRunAt(
+ params.monitor.schedule_cron,
+ new Date(),
+ params.monitor.schedule_timezone,
+ ).toISOString()
+ : null;
+ const { error } = await supabase_service
+ .from("monitors")
+ .update({
+ locked_at: null,
+ locked_until: null,
+ last_run_at: params.check.finished_at ?? new Date().toISOString(),
+ last_check_id: params.check.id,
+ next_run_at: nextRunAt,
+ last_check_summary: toMonitorSummary(params.check),
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", params.monitor.id);
+
+ throwIfError(error, "Failed to advance monitor after skipped check");
+}
+
+export async function getMonitorCheck(
+ teamId: string,
+ monitorId: string,
+ checkId: string,
+): Promise<MonitorCheckRow | null> {
+ const { data, error } = await supabase_rr_service
+ .from("monitor_checks")
+ .select("*")
+ .eq("id", checkId)
+ .eq("monitor_id", monitorId)
+ .eq("team_id", teamId)
+ .maybeSingle();
+
+ throwIfError(error, "Failed to get monitor check");
+ return data as MonitorCheckRow | null;
+}
+
+export async function listRunningMonitorChecks(
+ limit: number = 100,
+): Promise<MonitorCheckRow[]> {
+ const { data, error } = await supabase_service
+ .from("monitor_checks")
+ .select("*")
+ .eq("status", "running")
+ .order("created_at", { ascending: true })
+ .limit(limit);
+
+ throwIfError(error, "Failed to list running monitor checks");
+ return (data ?? []) as MonitorCheckRow[];
+}
+
+export async function listMonitorChecks(params: {
+ teamId: string;
+ monitorId: string;
+ limit: number;
+ offset: number;
+ status?: MonitorCheckRow["status"];
+}): Promise<MonitorCheckRow[]> {
+ let query = supabase_rr_service
+ .from("monitor_checks")
+ .select("*")
+ .eq("monitor_id", params.monitorId)
+ .eq("team_id", params.teamId)
+ .order("created_at", { ascending: false });
+
+ if (params.status) {
+ query = query.eq("status", params.status);
+ }
+
+ const { data, error } = await query.range(
+ params.offset,
+ params.offset + params.limit - 1,
+ );
+
+ throwIfError(error, "Failed to list monitor checks");
+ return (data ?? []) as MonitorCheckRow[];
+}
+
+export async function updateMonitorCheck(
+ checkId: string,
+ patch: Partial<MonitorCheckRow>,
+): Promise<MonitorCheckRow> {
+ const { data, error } = await supabase_service
+ .from("monitor_checks")
+ .update({
+ ...patch,
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", checkId)
+ .select("*")
+ .single();
+
+ throwIfError(error, "Failed to update monitor check");
+ return data as MonitorCheckRow;
+}
+
+export async function insertMonitorCheckPages(
+ pages: MonitorCheckPageInsert[],
+): Promise<void> {
+ if (pages.length === 0) return;
+
+ const { error } = await supabase_service.from("monitor_check_pages").insert(
+ pages.map(page => ({
+ id: uuidv7(),
+ ...page,
+ url_hash: page.url_hash ?? hashMonitorUrl(page.url),
+ })),
+ );
+
+ throwIfError(error, "Failed to insert monitor check pages");
+}
+
+export async function listMonitorCheckPages(params: {
+ teamId: string;
+ monitorId: string;
+ checkId: string;
+ limit: number;
+ skip: number;
+ status?: string;
+}): Promise<any[]> {
+ let query = supabase_rr_service
+ .from("monitor_check_pages")
+ .select("*")
+ .eq("check_id", params.checkId)
+ .eq("monitor_id", params.monitorId)
+ .eq("team_id", params.teamId)
+ .order("created_at", { ascending: true });
+
+ if (params.status) {
+ query = query.eq("status", params.status);
+ }
+
+ const { data, error } = await query.range(
+ params.skip,
+ params.skip + params.limit - 1,
+ );
+
+ throwIfError(error, "Failed to list monitor check pages");
+ return data ?? [];
+}
+
+export async function countMonitorCheckPages(params: {
+ checkId: string;
+ targetId?: string;
+ status?: string;
+}): Promise<number> {
+ let query = supabase_rr_service
+ .from("monitor_check_pages")
+ .select("id", { count: "exact", head: true })
+ .eq("check_id", params.checkId);
+
+ if (params.targetId) {
+ query = query.eq("target_id", params.targetId);
+ }
+ if (params.status) {
+ query = query.eq("status", params.status);
+ }
+
+ const { count, error } = await query;
+ throwIfError(error, "Failed to count monitor check pages");
+ return count ?? 0;
+}
+
+export async function getMonitorPage(params: {
+ monitorId: string;
+ targetId: string;
+ url: string;
+}): Promise<MonitorPageRow | null> {
+ const { data, error } = await supabase_rr_service
+ .from("monitor_pages")
+ .select("*")
+ .eq("monitor_id", params.monitorId)
+ .eq("target_id", params.targetId)
+ .eq("url_hash", hashMonitorUrl(params.url))
+ .maybeSingle();
+
+ throwIfError(error, "Failed to get monitor page");
+ return data as MonitorPageRow | null;
+}
+
+export async function upsertMonitorPage(params: {
+ monitorId: string;
+ teamId: string;
+ targetId: string;
+ url: string;
+ source: "explicit" | "discovered";
+ checkId: string;
+ scrapeId: string | null;
+ status: "same" | "new" | "changed" | "removed" | "error";
+ metadata?: unknown;
+}): Promise<void> {
+ const now = new Date().toISOString();
+
+ const existing = await getMonitorPage({
+ monitorId: params.monitorId,
+ targetId: params.targetId,
+ url: params.url,
+ });
+
+ if (!existing) {
+ const { error } = await supabase_service.from("monitor_pages").insert({
+ monitor_id: params.monitorId,
+ team_id: params.teamId,
+ target_id: params.targetId,
+ url: params.url,
+ url_hash: hashMonitorUrl(params.url),
+ source: params.source,
+ first_seen_check_id: params.checkId,
+ last_seen_check_id:
+ params.status === "removed" ? undefined : params.checkId,
+ last_changed_check_id:
+ params.status === "changed" || params.status === "new"
+ ? params.checkId
+ : undefined,
+ last_scrape_id: params.scrapeId,
+ last_status: params.status,
+ is_removed: params.status === "removed",
+ removed_at: params.status === "removed" ? now : null,
+ metadata: params.metadata ?? null,
+ created_at: now,
+ updated_at: now,
+ });
+
+ throwIfError(error, "Failed to insert monitor page");
+ return;
+ }
+
+ const patch: Record<string, unknown> = {
+ last_status: params.status,
+ is_removed: params.status === "removed",
+ removed_at: params.status === "removed" ? now : null,
+ metadata: params.metadata ?? existing.metadata ?? null,
+ updated_at: now,
+ };
+ if (params.status !== "removed") {
+ patch.last_seen_check_id = params.checkId;
+ patch.last_scrape_id = params.scrapeId;
+ }
+ if (params.status === "changed" || params.status === "new") {
+ patch.last_changed_check_id = params.checkId;
+ }
+
+ const { error } = await supabase_service
+ .from("monitor_pages")
+ .update(patch)
+ .eq("id", existing.id);
+
+ throwIfError(error, "Failed to update monitor page");
+}
+
+export async function listActiveMonitorPages(params: {
+ monitorId: string;
+ targetId: string;
+}): Promise<MonitorPageRow[]> {
+ const pageSize = 1000;
+ const pages: MonitorPageRow[] = [];
+ let offset = 0;
+
+ while (true) {
+ const { data, error } = await supabase_rr_service
+ .from("monitor_pages")
+ .select("*")
+ .eq("monitor_id", params.monitorId)
+ .eq("target_id", params.targetId)
+ .eq("is_removed", false)
+ .order("created_at", { ascending: true })
+ .range(offset, offset + pageSize - 1);
+
+ throwIfError(error, "Failed to list active monitor pages");
+ const batch = (data ?? []) as MonitorPageRow[];
+ pages.push(...batch);
+ if (batch.length < pageSize) break;
+ offset += pageSize;
+ }
+
+ return pages;
+}
+
+export async function claimDueMonitors(params: {
+ workerId: string;
+ limit: number;
+ leaseSeconds: number;
+}): Promise<MonitorRow[]> {
+ const { data, error } = await supabase_service.rpc(
+ "monitoring_claim_due_monitors",
+ {
+ p_worker_id: params.workerId,
+ p_limit: params.limit,
+ p_lease_seconds: params.leaseSeconds,
+ },
+ );
+
+ throwIfError(error, "Failed to claim due monitors");
+ return (data ?? []) as MonitorRow[];
+}
diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.ts
new file mode 100644
index 000000000..7a1aab521
--- /dev/null
+++ b/apps/api/src/services/monitoring/types.ts
@@ -0,0 +1,275 @@
+import { z } from "zod";
+import {
+ crawlerOptions,
+ URL as urlSchema,
+ type ScrapeOptions,
+} from "../../controllers/v2/types";
+import { createWebhookSchema } from "../webhook/schema";
+import { parseMonitorScheduleText } from "./cron";
+
+const formatSchema = z.union([z.string(), z.record(z.string(), z.unknown())]);
+
+const scrapeOptionsSchema = z
+ .object({
+ formats: z.array(formatSchema).optional(),
+ })
+ .catchall(z.unknown())
+ .optional()
+ .default({});
+
+const scrapeTargetSchema = z.strictObject({
+ id: z.string().uuid().optional(),
+ type: z.literal("scrape"),
+ urls: z.array(urlSchema).min(1),
+ scrapeOptions: scrapeOptionsSchema,
+});
+
+const crawlTargetSchema = z.strictObject({
+ id: z.string().uuid().optional(),
+ type: z.literal("crawl"),
+ url: urlSchema,
+ crawlOptions: crawlerOptions
+ .partial()
+ .catchall(z.unknown())
+ .optional()
+ .default({}),
+ scrapeOptions: scrapeOptionsSchema,
+});
+
+const monitorTargetSchema = z.union([scrapeTargetSchema, crawlTargetSchema]);
+
+const monitorWebhookSchema = createWebhookSchema([
+ "monitor.page",
+ "monitor.check.completed",
+]);
+
+const monitorScheduleSchema = z
+ .strictObject({
+ cron: z.string().min(1).max(128).optional(),
+ text: z.string().min(1).max(128).optional(),
+ timezone: z.string().min(1).max(128).optional().default("UTC"),
+ })
+ .superRefine((schedule, ctx) => {
+ if (!schedule.cron && !schedule.text) {
+ ctx.addIssue({
+ code: "custom",
+ message: "Schedule must include either cron or text",
+ path: ["cron"],
+ });
+ }
+ if (schedule.cron && schedule.text) {
+ ctx.addIssue({
+ code: "custom",
+ message: "Schedule must include either cron or text, not both",
+ path: ["text"],
+ });
+ }
+ if (schedule.text) {
+ try {
+ parseMonitorScheduleText(schedule.text);
+ } catch (error) {
+ ctx.addIssue({
+ code: "custom",
+ message: error instanceof Error ? error.message : String(error),
+ path: ["text"],
+ });
+ }
+ }
+ })
+ .transform(schedule => ({
+ cron: schedule.cron ?? parseMonitorScheduleText(schedule.text!),
+ timezone: schedule.timezone,
+ }));
+
+const monitorNotificationSchema = z
+ .strictObject({
+ email: z
+ .strictObject({
+ enabled: z.boolean().optional().default(false),
+ recipients: z.array(z.email()).max(25).optional().default([]),
+ includeDiffs: z.boolean().optional().default(false),
+ })
+ .optional(),
+ })
+ .optional()
+ .default({});
+
+export const createMonitorSchema = z.strictObject({
+ name: z.string().min(1).max(256),
+ schedule: monitorScheduleSchema,
+ webhook: monitorWebhookSchema.optional(),
+ notification: monitorNotificationSchema,
+ targets: z.array(monitorTargetSchema).min(1).max(50),
+ retentionDays: z.number().int().positive().max(365).optional().default(30),
+});
+
+export const updateMonitorSchema = createMonitorSchema
+ .partial()
+ .extend({
+ status: z.enum(["active", "paused"]).optional(),
+ })
+ .refine(x => Object.keys(x).length > 0, "Update body cannot be empty");
+
+export const listMonitorsQuerySchema = z.object({
+ limit: z.coerce.number().int().positive().max(100).optional().default(25),
+ offset: z.coerce.number().int().nonnegative().optional().default(0),
+});
+
+export const listMonitorChecksQuerySchema = z.object({
+ limit: z.coerce.number().int().positive().max(100).optional().default(25),
+ offset: z.coerce.number().int().nonnegative().optional().default(0),
+ status: z
+ .enum([
+ "queued",
+ "running",
+ "completed",
+ "failed",
+ "partial",
+ "skipped_overlap",
+ ])
+ .optional(),
+});
+
+export const monitorCheckDetailQuerySchema = z.object({
+ limit: z.coerce.number().int().positive().max(100).optional().default(25),
+ skip: z.coerce.number().int().nonnegative().optional().default(0),
+ status: z.enum(["same", "new", "changed", "removed", "error"]).optional(),
+});
+
+export type MonitorTarget = z.infer<typeof monitorTargetSchema> & {
+ id: string;
+};
+export type CreateMonitorRequest = z.infer<typeof createMonitorSchema>;
+export type UpdateMonitorRequest = z.infer<typeof updateMonitorSchema>;
+type MonitorNotification = z.infer<typeof monitorNotificationSchema>;
+
+export type MonitorRow = {
+ id: string;
+ team_id: string;
+ name: string;
+ status: "active" | "paused" | "deleted";
+ schedule_cron: string;
+ schedule_timezone: string;
+ next_run_at: string | null;
+ last_run_at: string | null;
+ current_check_id: string | null;
+ locked_at: string | null;
+ locked_until: string | null;
+ retention_days: number;
+ estimated_credits_per_month: number | null;
+ targets: MonitorTarget[];
+ webhook: unknown | null;
+ notification: MonitorNotification | null;
+ last_check_summary: MonitorSummary | null;
+ created_at: string;
+ updated_at: string;
+ deleted_at: string | null;
+};
+
+export type MonitorCheckRow = {
+ id: string;
+ monitor_id: string;
+ team_id: string;
+ trigger: "scheduled" | "manual";
+ status:
+ | "queued"
+ | "running"
+ | "completed"
+ | "failed"
+ | "partial"
+ | "skipped_overlap";
+ scheduled_for: string | null;
+ started_at: string | null;
+ finished_at: string | null;
+ estimated_credits: number | null;
+ reserved_credits: number | null;
+ actual_credits: number | null;
+ autumn_lock_id: string | null;
+ billing_status:
+ | "not_applicable"
+ | "reserved"
+ | "confirmed"
+ | "released"
+ | "failed";
+ total_pages: number;
+ same_count: number;
+ changed_count: number;
+ new_count: number;
+ removed_count: number;
+ error_count: number;
+ target_results: unknown | null;
+ webhook_payload: unknown | null;
+ email_payload: unknown | null;
+ notification_status: unknown | null;
+ error: string | null;
+ created_at: string;
+ updated_at: string;
+};
+
+type MonitorPageStatus = "same" | "new" | "changed" | "removed" | "error";
+type MonitorPageSource = "explicit" | "discovered";
+
+export type MonitorPageRow = {
+ id: string;
+ monitor_id: string;
+ team_id: string;
+ target_id: string;
+ url: string;
+ url_hash: string;
+ source: MonitorPageSource;
+ first_seen_check_id: string | null;
+ last_seen_check_id: string | null;
+ last_changed_check_id: string | null;
+ last_scrape_id: string | null;
+ last_status: MonitorPageStatus;
+ is_removed: boolean;
+ removed_at: string | null;
+ metadata: unknown | null;
+ created_at: string;
+ updated_at: string;
+};
+
+export type MonitorSummary = {
+ totalPages: number;
+ same: number;
+ changed: number;
+ new: number;
+ removed: number;
+ error: number;
+};
+
+export type MonitorCheckPageInsert = {
+ check_id: string;
+ monitor_id: string;
+ team_id: string;
+ target_id: string;
+ url: string;
+ url_hash?: string;
+ status: MonitorPageStatus;
+ previous_scrape_id?: string | null;
+ current_scrape_id?: string | null;
+ diff_gcs_key?: string | null;
+ diff_text_bytes?: number | null;
+ diff_json_bytes?: number | null;
+ status_code?: number | null;
+ error?: string | null;
+ metadata?: unknown | null;
+};
+
+export function withMarkdownFormat(
+ options: Record<string, unknown>,
+): ScrapeOptions {
+ const formats = Array.isArray(options.formats) ? options.formats : [];
+ const hasMarkdown = formats.some(format =>
+ typeof format === "string"
+ ? format === "markdown"
+ : typeof format === "object" &&
+ format !== null &&
+ (format as any).type === "markdown",
+ );
+
+ return {
+ ...options,
+ formats: hasMarkdown ? formats : ["markdown", ...formats],
+ } as ScrapeOptions;
+}
diff --git a/apps/api/src/services/notification/monitoring_email.ts b/apps/api/src/services/notification/monitoring_email.ts
new file mode 100644
index 000000000..06d90ed83
--- /dev/null
+++ b/apps/api/src/services/notification/monitoring_email.ts
@@ -0,0 +1,216 @@
+import { Resend } from "resend";
+import escapeHtml from "escape-html";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+import { supabase_service } from "../supabase";
+import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";
+
+const logger = _logger.child({ module: "monitoring-email" });
+
+type MonitoringEmailPage = {
+ url: string;
+ status: string;
+ error?: string | null;
+};
+
+type MonitoringEmailPayload = {
+ monitorId: string;
+ monitorName: string;
+ checkId: string;
+ summary: {
+ changed: number;
+ new: number;
+ removed: number;
+ error: number;
+ totalPages: number;
+ };
+ pages: MonitoringEmailPage[];
+ creditsUsed: number | null;
+};
+
+async function getTeamEmails(teamId: string): Promise<string[]> {
+ const { data, error } = await supabase_service
+ .from("user_teams")
+ .select(
+ "users(email, id, notification_preferences(unsubscribed_all, email_preferences))",
+ )
+ .eq("team_id", teamId);
+
+ if (error) {
+ logger.warn("Failed to load team emails", { error, teamId });
+ return [];
+ }
+
+ const emails = new Set<string>();
+ for (const row of data ?? []) {
+ const user = (row as any).users;
+ const email = user?.email;
+ if (!email) continue;
+
+ const prefs = Array.isArray(user.notification_preferences)
+ ? user.notification_preferences[0]
+ : user.notification_preferences;
+ if (prefs?.unsubscribed_all) continue;
+ if (
+ Array.isArray(prefs?.email_preferences) &&
+ !prefs.email_preferences.includes("system_alerts")
+ ) {
+ continue;
+ }
+
+ emails.add(email);
+ }
+
+ return [...emails];
+}
+
+function buildHtml(payload: MonitoringEmailPayload): string {
+ const pageItems = payload.pages
+ .slice(0, 20)
+ .map(page => {
+ const url = escapeHtml(page.url);
+ return `<li><strong>${escapeHtml(page.status)}</strong>: <a href="${url}">${url}</a>${
+ page.error ? ` &mdash; ${escapeHtml(page.error)}` : ""
+ }</li>`;
+ })
+ .join("");
+
+ return `Hey there,<br/>
+<p>Your Firecrawl monitor <strong>${escapeHtml(payload.monitorName)}</strong> detected activity.</p>
+<ul>
+ <li>Changed: ${payload.summary.changed}</li>
+ <li>New: ${payload.summary.new}</li>
+ <li>Removed: ${payload.summary.removed}</li>
+ <li>Errors: ${payload.summary.error}</li>
+ <li>Total pages checked: ${payload.summary.totalPages}</li>
+</ul>
+${pageItems ? `<p>Top pages:</p><ul>${pageItems}</ul>` : ""}
+<p>Check ID: <code>${escapeHtml(payload.checkId)}</code></p>
+<p>Credits used: ${payload.creditsUsed ?? "unknown"}</p>
+<br/>Thanks,<br/>Firecrawl Team<br/>`;
+}
+
+export async function sendMonitoringEmailSummary(params: {
+ monitor: MonitorRow;
+ check: MonitorCheckRow;
+ pages: MonitoringEmailPage[];
+}): Promise<{
+ attempted: boolean;
+ success: boolean;
+ recipients: string[];
+ error?: string;
+}> {
+ const configEmail = params.monitor.notification?.email;
+ if (!configEmail?.enabled) {
+ logger.info(
+ "Skipping monitoring email summary; email notifications disabled",
+ {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ },
+ );
+ return { attempted: false, success: true, recipients: [] };
+ }
+
+ if (
+ params.check.changed_count +
+ params.check.new_count +
+ params.check.removed_count +
+ params.check.error_count <=
+ 0
+ ) {
+ logger.info("Skipping monitoring email summary; no changes detected", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ changed: params.check.changed_count,
+ new: params.check.new_count,
+ removed: params.check.removed_count,
+ errors: params.check.error_count,
+ });
+ return { attempted: false, success: true, recipients: [] };
+ }
+
+ const explicitRecipients = configEmail.recipients ?? [];
+ const teamRecipients =
+ explicitRecipients.length > 0
+ ? []
+ : await getTeamEmails(params.monitor.team_id);
+ const recipients = [...new Set([...explicitRecipients, ...teamRecipients])];
+ if (recipients.length === 0) {
+ logger.info("Skipping monitoring email summary; no recipients configured", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ });
+ return { attempted: false, success: true, recipients };
+ }
+
+ const resendApiKey = config.RESEND_API_KEY?.trim();
+ if (!resendApiKey) {
+ logger.warn(
+ "Skipping monitoring email summary; RESEND_API_KEY is not set",
+ {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ recipients,
+ },
+ );
+ return { attempted: false, success: true, recipients };
+ }
+
+ const payload: MonitoringEmailPayload = {
+ monitorId: params.monitor.id,
+ monitorName: params.monitor.name,
+ checkId: params.check.id,
+ summary: {
+ changed: params.check.changed_count,
+ new: params.check.new_count,
+ removed: params.check.removed_count,
+ error: params.check.error_count,
+ totalPages: params.check.total_pages,
+ },
+ pages: params.pages,
+ creditsUsed: params.check.actual_credits,
+ };
+
+ const resend = new Resend(resendApiKey);
+ try {
+ const { error } = await resend.emails.send({
+ from: "Firecrawl <[email protected]>",
+ to: recipients,
+ reply_to: "[email protected]",
+ subject: `Monitor changes detected: ${params.monitor.name}`,
+ html: buildHtml(payload),
+ });
+
+ if (error) {
+ logger.warn("Monitoring email summary send failed", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ recipients,
+ error,
+ });
+ return {
+ attempted: true,
+ success: false,
+ recipients,
+ error: typeof error === "string" ? error : JSON.stringify(error),
+ };
+ }
+
+ logger.info("Monitoring email summary sent", {
+ monitorId: params.monitor.id,
+ checkId: params.check.id,
+ recipients,
+ });
+
+ return { attempted: true, success: true, recipients };
+ } catch (error) {
+ logger.warn("Failed to send monitoring email summary", { error });
+ return {
+ attempted: true,
+ success: false,
+ recipients,
+ error: error instanceof Error ? error.message : String(error),
+ };
+ }
+}
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 0afc8fce8..289a544e9 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -25,6 +25,12 @@ import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
import { TransportableError } from "../lib/error";
+import {
+ processMonitorCheckJob,
+ reconcileRunningMonitorChecks,
+} from "./monitoring/runner";
+import { enqueueDueMonitorChecks } from "./monitoring/scheduler";
+import { consumeMonitorCheckJobs } from "./monitoring/queue";
configDotenv();
@@ -38,6 +44,7 @@ const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;
const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;
const runningJobs: Set<string> = new Set();
+let monitorSchedulerInterval: NodeJS.Timeout | null = null;
const processDeepResearchJobInternal = async (
token: string,
@@ -407,7 +414,7 @@ let currentLiveness: boolean = true;
app.get("/liveness", (req, res) => {
_logger.info("Liveness endpoint hit");
- if (config.USE_DB_AUTHENTICATION) {
+ if (config.USE_DB_AUTHENTICATION && config.NUQ_RABBITMQ_URL) {
// networking check for Kubernetes environments
const host = config.FIRECRAWL_APP_HOST;
const port = config.FIRECRAWL_APP_PORT;
@@ -452,12 +459,43 @@ app.listen(workerPort, () => {
initializeEngineForcing();
+ if (config.USE_DB_AUTHENTICATION) {
+ monitorSchedulerInterval = setInterval(() => {
+ enqueueDueMonitorChecks().catch(error => {
+ _logger.error("Failed to enqueue due monitor checks", { error });
+ });
+ reconcileRunningMonitorChecks().catch(error => {
+ _logger.error("Failed to reconcile running monitor checks", { error });
+ });
+ }, 60_000);
+ enqueueDueMonitorChecks().catch(error => {
+ _logger.error("Failed to enqueue due monitor checks", { error });
+ });
+ reconcileRunningMonitorChecks().catch(error => {
+ _logger.error("Failed to reconcile running monitor checks", { error });
+ });
+
+ await consumeMonitorCheckJobs(processMonitorCheckJob);
+ } else if (!config.USE_DB_AUTHENTICATION) {
+ _logger.info(
+ "Skipping monitor worker startup because database authentication is disabled",
+ );
+ } else {
+ _logger.info(
+ "Skipping monitor worker startup because NUQ_RABBITMQ_URL is not configured",
+ );
+ }
+
await Promise.all([
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
crawlFinishWorker(),
]);
+ if (monitorSchedulerInterval) {
+ clearInterval(monitorSchedulerInterval);
+ }
+
_logger.info("All workers exited. Waiting for all jobs to finish...");
while (runningJobs.size > 0) {
diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.ts
index a3fbdabde..550c4f430 100644
--- a/apps/api/src/services/webhook/types.ts
+++ b/apps/api/src/services/webhook/types.ts
@@ -13,6 +13,8 @@ export enum WebhookEvent {
EXTRACT_STARTED = "extract.started",
EXTRACT_COMPLETED = "extract.completed",
EXTRACT_FAILED = "extract.failed",
+ MONITOR_PAGE = "monitor.page",
+ MONITOR_CHECK_COMPLETED = "monitor.check.completed",
}
export type WebhookEventDataMap = {
@@ -25,6 +27,8 @@ export type WebhookEventDataMap = {
[WebhookEvent.EXTRACT_STARTED]: ExtractStartedData;
[WebhookEvent.EXTRACT_COMPLETED]: ExtractCompletedData;
[WebhookEvent.EXTRACT_FAILED]: ExtractFailedData;
+ [WebhookEvent.MONITOR_PAGE]: MonitorPageData;
+ [WebhookEvent.MONITOR_CHECK_COMPLETED]: MonitorCheckCompletedData;
};
export type WebhookConfig = z.infer<typeof webhookSchema>;
@@ -113,3 +117,36 @@ interface ExtractFailedData extends BaseWebhookData {
success: false;
error: string;
}
+
+// monitor
+interface MonitorPageData extends BaseWebhookData {
+ success: boolean;
+ data: {
+ monitorId: string;
+ checkId: string;
+ url: string;
+ status: string;
+ previousScrapeId?: string | null;
+ currentScrapeId?: string | null;
+ error?: string | null;
+ };
+ error?: string;
+}
+
+interface MonitorCheckCompletedData extends BaseWebhookData {
+ success: boolean;
+ data: {
+ monitorId: string;
+ checkId: string;
+ status: string;
+ summary: {
+ totalPages: number;
+ same: number;
+ changed: number;
+ new: number;
+ removed: number;
+ error: number;
+ };
+ };
+ error?: string;
+}
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 35516137f..d35926142 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -73,6 +73,8 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
),
zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,
cancelled: sc.cancelled ?? false,
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
},
false,
);
@@ -158,6 +160,8 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
credits_cost: credits_billed ?? 0,
zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,
cancelled: sc.cancelled ?? false,
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
},
false,
);
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 2abada584..e6ff468da 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -83,6 +83,10 @@ import {
import { ScrapeUrlResponse } from "../../scraper/scrapeURL";
import { logScrape } from "../logging/log_job";
import { FeatureFlag } from "../../scraper/scrapeURL/engines";
+import {
+ recordMonitorScrapeFailure,
+ recordMonitorScrapeSuccess,
+} from "../monitoring/results";
configDotenv();
@@ -456,6 +460,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
v1: job.data.v1,
zeroDataRetention: job.data.zeroDataRetention,
apiKeyId: job.data.apiKeyId,
+ monitoring: job.data.monitoring
+ ? { ...job.data.monitoring, source: "discovered" }
+ : undefined,
},
jobId,
jobPriority,
@@ -529,6 +536,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
skipNuq: job.data.skipNuq ?? false,
is_parse: Boolean(job.data.internalOptions?.isParse),
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
},
true,
);
@@ -576,6 +585,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
+ await recordMonitorScrapeSuccess(job, doc).catch(error =>
+ logger.warn("Failed to record monitor scrape result", { error }),
+ );
+
logger.debug("Declaring job as done...");
await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
} else {
@@ -613,6 +626,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
skipNuq: job.data.skipNuq ?? false,
is_parse: Boolean(job.data.internalOptions?.isParse),
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
},
false,
);
@@ -630,6 +645,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
}).catch(err => logger.warn("Scrape tracking failed", { error: err }));
+ await recordMonitorScrapeSuccess(job, doc).catch(error =>
+ logger.warn("Failed to record monitor scrape result", { error }),
+ );
+
if (job.data.skipNuq) {
// doesn't use GCS for result retrieval, safe to not await
logScrapePromise.catch(err =>
@@ -789,6 +808,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
skipNuq: job.data.skipNuq ?? false,
is_parse: Boolean(job.data.internalOptions?.isParse),
+ monitor_id: job.data.monitoring?.monitorId,
+ monitor_check_id: job.data.monitoring?.checkId,
},
true,
);
@@ -806,6 +827,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
}).catch(err => logger.warn("Scrape tracking failed", { error: err }));
+ await recordMonitorScrapeFailure(job, error).catch(err =>
+ logger.warn("Failed to record monitor scrape failure", { error: err }),
+ );
+
return data;
} finally {
if (abortTimeoutHandle) clearTimeout(abortTimeoutHandle);
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index b5d4815fb..8ad6ffbee 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -21,6 +21,12 @@ type ScrapeJobCommon = {
traceContext?: SerializedTraceContext;
skipNuq?: boolean;
requestId?: string;
+ monitoring?: {
+ monitorId: string;
+ checkId: string;
+ targetId: string;
+ source: "explicit" | "discovered";
+ };
};
export type ScrapeJobData = ScrapeJobCommon &
diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
index e2a0f936f..4962ae390 100644
--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj
@@ -8,7 +8,7 @@
<!-- NuGet package metadata -->
<PackageId>firecrawl-sdk</PackageId>
- <Version>1.2.1</Version>
+ <Version>1.3.0</Version>
<Authors>Firecrawl</Authors>
<Company>Firecrawl</Company>
<Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>
diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
index 1f2ae2b78..11c5da6a3 100644
--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs
@@ -1,6 +1,7 @@
using System.Text.Json;
using Firecrawl.Exceptions;
using Firecrawl.Models;
+using MonitorModel = Firecrawl.Models.Monitor;
namespace Firecrawl;
@@ -321,6 +322,122 @@ public class FirecrawlClient
return response.Data ?? throw new FirecrawlException("Map response contained no data");
}
+ // ================================================================
+ // MONITOR
+ // ================================================================
+
+ public async Task<MonitorModel> CreateMonitorAsync(
+ CreateMonitorRequest request,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(request);
+
+ var response = await _http.PostAsync<ApiResponse<MonitorModel>>(
+ "/v2/monitor", request, cancellationToken: cancellationToken);
+
+ return response.Data ?? throw new FirecrawlException("Create monitor response contained no data");
+ }
+
+ public async Task<List<MonitorModel>> ListMonitorsAsync(
+ int? limit = null,
+ int? offset = null,
+ CancellationToken cancellationToken = default)
+ {
+ var response = await _http.GetAsync<ApiResponse<List<MonitorModel>>>(
+ $"/v2/monitor{BuildQuery(limit, offset)}", cancellationToken);
+
+ return response.Data ?? new List<MonitorModel>();
+ }
+
+ public async Task<MonitorModel> GetMonitorAsync(
+ string monitorId,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(monitorId);
+
+ var response = await _http.GetAsync<ApiResponse<MonitorModel>>(
+ $"/v2/monitor/{monitorId}", cancellationToken);
+
+ return response.Data ?? throw new FirecrawlException("Get monitor response contained no data");
+ }
+
+ public async Task<MonitorModel> UpdateMonitorAsync(
+ string monitorId,
+ UpdateMonitorRequest request,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(monitorId);
+ ArgumentNullException.ThrowIfNull(request);
+
+ var response = await _http.PatchAsync<ApiResponse<MonitorModel>>(
+ $"/v2/monitor/{monitorId}", request, cancellationToken);
+
+ return response.Data ?? throw new FirecrawlException("Update monitor response contained no data");
+ }
+
+ public async Task<bool> DeleteMonitorAsync(
+ string monitorId,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(monitorId);
+
+ var response = await _http.DeleteAsync<Dictionary<string, object>>(
+ $"/v2/monitor/{monitorId}", cancellationToken);
+
+ return response.TryGetValue("success", out var success) && success switch
+ {
+ bool value => value,
+ JsonElement element when element.ValueKind == JsonValueKind.True => true,
+ _ => false
+ };
+ }
+
+ public async Task<MonitorCheck> RunMonitorAsync(
+ string monitorId,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(monitorId);
+
+ var response = await _http.PostAsync<ApiResponse<MonitorCheck>>(
+ $"/v2/monitor/{monitorId}/run", new Dictionary<string, object>(), cancellationToken: cancellationToken);
+
+ return response.Data ?? throw new FirecrawlException("Run monitor response contained no data");
+ }
+
+ public async Task<List<MonitorCheck>> ListMonitorChecksAsync(
+ string monitorId,
+ int? limit = null,
+ int? offset = null,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(monitorId);
+
+ var response = await _http.GetAsync<ApiResponse<List<MonitorCheck>>>(
+ $"/v2/monitor/{monitorId}/checks{BuildQuery(limit, offset)}", cancellationToken);
+
+ return response.Data ?? new List<MonitorCheck>();
+ }
+
+ public async Task<MonitorCheckDetail> GetMonitorCheckAsync(
+ string monitorId,
+ string checkId,
+ int? limit = null,
+ int? skip = null,
+ string? status = null,
+ bool autoPaginate = true,
+ CancellationToken cancellationToken = default)
+ {
+ ArgumentNullException.ThrowIfNull(monitorId);
+ ArgumentNullException.ThrowIfNull(checkId);
+
+ var response = await _http.GetAsync<ApiResponse<MonitorCheckDetail>>(
+ $"/v2/monitor/{monitorId}/checks/{checkId}{BuildMonitorCheckQuery(limit, skip, status)}",
+ cancellationToken);
+
+ var check = response.Data ?? throw new FirecrawlException("Get monitor check response contained no data");
+ return autoPaginate ? await PaginateMonitorCheckAsync(check, cancellationToken) : check;
+ }
+
// ================================================================
// SEARCH
// ================================================================
@@ -460,6 +577,32 @@ public class FirecrawlClient
return job;
}
+ private async Task<MonitorCheckDetail> PaginateMonitorCheckAsync(
+ MonitorCheckDetail check,
+ CancellationToken cancellationToken)
+ {
+ check.Pages ??= new List<MonitorCheckPage>();
+ var current = check;
+
+ while (!string.IsNullOrEmpty(current.Next))
+ {
+ var response = await _http.GetAbsoluteAsync<ApiResponse<MonitorCheckDetail>>(
+ current.Next, cancellationToken);
+ if (response.Data == null)
+ break;
+
+ var nextPage = response.Data;
+
+ if (nextPage.Pages is { Count: > 0 })
+ check.Pages.AddRange(nextPage.Pages);
+
+ current = nextPage;
+ }
+
+ check.Next = null;
+ return check;
+ }
+
// ================================================================
// INTERNAL UTILITIES
// ================================================================
@@ -474,6 +617,32 @@ public class FirecrawlClient
?? new Dictionary<string, object>();
}
+ private static string BuildQuery(int? limit = null, int? offset = null, string? status = null)
+ {
+ var query = new List<string>();
+ if (limit.HasValue)
+ query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");
+ if (offset.HasValue)
+ query.Add($"offset={Uri.EscapeDataString(offset.Value.ToString())}");
+ if (!string.IsNullOrWhiteSpace(status))
+ query.Add($"status={Uri.EscapeDataString(status)}");
+
+ return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
+ }
+
+ private static string BuildMonitorCheckQuery(int? limit = null, int? skip = null, string? status = null)
+ {
+ var query = new List<string>();
+ if (limit.HasValue)
+ query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");
+ if (skip.HasValue)
+ query.Add($"skip={Uri.EscapeDataString(skip.Value.ToString())}");
+ if (!string.IsNullOrWhiteSpace(status))
+ query.Add($"status={Uri.EscapeDataString(status)}");
+
+ return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);
+ }
+
private static string ResolveApiKey(string? apiKey)
{
if (!string.IsNullOrWhiteSpace(apiKey))
diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs
index 2da99b16e..a7ba80dbf 100644
--- a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs
+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs
@@ -139,6 +139,25 @@ internal class FirecrawlHttpClient
return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
}
+ internal async Task<T> PatchAsync<T>(
+ string path,
+ object body,
+ CancellationToken cancellationToken = default)
+ {
+ var url = _baseUrl + path;
+ var json = JsonSerializer.Serialize(body, JsonOptions);
+
+ HttpRequestMessage BuildRequest()
+ {
+ var content = new StringContent(json, Encoding.UTF8, "application/json");
+ var request = new HttpRequestMessage(HttpMethod.Patch, url) { Content = content };
+ ApplyStandardHeaders(request);
+ return request;
+ }
+
+ return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);
+ }
+
internal async Task<T> GetAbsoluteAsync<T>(string absoluteUrl, CancellationToken cancellationToken = default)
{
// Validate that the pagination URL belongs to the same host to prevent API key exfiltration
diff --git a/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs
new file mode 100644
index 000000000..ed5c7ef83
--- /dev/null
+++ b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs
@@ -0,0 +1,225 @@
+using System.Text.Json.Serialization;
+
+namespace Firecrawl.Models;
+
+public class MonitorSchedule
+{
+ [JsonPropertyName("cron")]
+ public string? Cron { get; set; }
+
+ [JsonPropertyName("timezone")]
+ public string? Timezone { get; set; }
+}
+
+public class CreateMonitorRequest
+{
+ [JsonPropertyName("name")]
+ public string? Name { get; set; }
+
+ [JsonPropertyName("schedule")]
+ public MonitorSchedule? Schedule { get; set; }
+
+ [JsonPropertyName("targets")]
+ public List<Dictionary<string, object>>? Targets { get; set; }
+
+ [JsonPropertyName("webhook")]
+ public Dictionary<string, object>? Webhook { get; set; }
+
+ [JsonPropertyName("notification")]
+ public Dictionary<string, object>? Notification { get; set; }
+
+ [JsonPropertyName("retentionDays")]
+ public int? RetentionDays { get; set; }
+}
+
+public class UpdateMonitorRequest
+{
+ [JsonPropertyName("name")]
+ public string? Name { get; set; }
+
+ [JsonPropertyName("status")]
+ public string? Status { get; set; }
+
+ [JsonPropertyName("schedule")]
+ public MonitorSchedule? Schedule { get; set; }
+
+ [JsonPropertyName("targets")]
+ public List<Dictionary<string, object>>? Targets { get; set; }
+
+ [JsonPropertyName("webhook")]
+ public Dictionary<string, object>? Webhook { get; set; }
+
+ [JsonPropertyName("notification")]
+ public Dictionary<string, object>? Notification { get; set; }
+
+ [JsonPropertyName("retentionDays")]
+ public int? RetentionDays { get; set; }
+}
+
+public class MonitorSummary
+{
+ [JsonPropertyName("totalPages")]
+ public int TotalPages { get; set; }
+
+ [JsonPropertyName("same")]
+ public int Same { get; set; }
+
+ [JsonPropertyName("changed")]
+ public int Changed { get; set; }
+
+ [JsonPropertyName("new")]
+ public int New { get; set; }
+
+ [JsonPropertyName("removed")]
+ public int Removed { get; set; }
+
+ [JsonPropertyName("error")]
+ public int Error { get; set; }
+}
+
+public class Monitor
+{
+ [JsonPropertyName("id")]
+ public string? Id { get; set; }
+
+ [JsonPropertyName("name")]
+ public string? Name { get; set; }
+
+ [JsonPropertyName("status")]
+ public string? Status { get; set; }
+
+ [JsonPropertyName("schedule")]
+ public MonitorSchedule? Schedule { get; set; }
+
+ [JsonPropertyName("nextRunAt")]
+ public string? NextRunAt { get; set; }
+
+ [JsonPropertyName("lastRunAt")]
+ public string? LastRunAt { get; set; }
+
+ [JsonPropertyName("currentCheckId")]
+ public string? CurrentCheckId { get; set; }
+
+ [JsonPropertyName("targets")]
+ public List<Dictionary<string, object>>? Targets { get; set; }
+
+ [JsonPropertyName("webhook")]
+ public Dictionary<string, object>? Webhook { get; set; }
+
+ [JsonPropertyName("notification")]
+ public Dictionary<string, object>? Notification { get; set; }
+
+ [JsonPropertyName("retentionDays")]
+ public int RetentionDays { get; set; }
+
+ [JsonPropertyName("estimatedCreditsPerMonth")]
+ public int? EstimatedCreditsPerMonth { get; set; }
+
+ [JsonPropertyName("lastCheckSummary")]
+ public MonitorSummary? LastCheckSummary { get; set; }
+
+ [JsonPropertyName("createdAt")]
+ public string? CreatedAt { get; set; }
+
+ [JsonPropertyName("updatedAt")]
+ public string? UpdatedAt { get; set; }
+}
+
+public class MonitorCheck
+{
+ [JsonPropertyName("id")]
+ public string? Id { get; set; }
+
+ [JsonPropertyName("monitorId")]
+ public string? MonitorId { get; set; }
+
+ [JsonPropertyName("status")]
+ public string? Status { get; set; }
+
+ [JsonPropertyName("trigger")]
+ public string? Trigger { get; set; }
+
+ [JsonPropertyName("scheduledFor")]
+ public string? ScheduledFor { get; set; }
+
+ [JsonPropertyName("startedAt")]
+ public string? StartedAt { get; set; }
+
+ [JsonPropertyName("finishedAt")]
+ public string? FinishedAt { get; set; }
+
+ [JsonPropertyName("estimatedCredits")]
+ public int? EstimatedCredits { get; set; }
+
+ [JsonPropertyName("reservedCredits")]
+ public int? ReservedCredits { get; set; }
+
+ [JsonPropertyName("actualCredits")]
+ public int? ActualCredits { get; set; }
+
+ [JsonPropertyName("billingStatus")]
+ public string? BillingStatus { get; set; }
+
+ [JsonPropertyName("summary")]
+ public MonitorSummary? Summary { get; set; }
+
+ [JsonPropertyName("targetResults")]
+ public object? TargetResults { get; set; }
+
+ [JsonPropertyName("notificationStatus")]
+ public object? NotificationStatus { get; set; }
+
+ [JsonPropertyName("error")]
+ public string? Error { get; set; }
+
+ [JsonPropertyName("createdAt")]
+ public string? CreatedAt { get; set; }
+
+ [JsonPropertyName("updatedAt")]
+ public string? UpdatedAt { get; set; }
+}
+
+public class MonitorCheckPage
+{
+ [JsonPropertyName("id")]
+ public string? Id { get; set; }
+
+ [JsonPropertyName("targetId")]
+ public string? TargetId { get; set; }
+
+ [JsonPropertyName("url")]
+ public string? Url { get; set; }
+
+ [JsonPropertyName("status")]
+ public string? Status { get; set; }
+
+ [JsonPropertyName("previousScrapeId")]
+ public string? PreviousScrapeId { get; set; }
+
+ [JsonPropertyName("currentScrapeId")]
+ public string? CurrentScrapeId { get; set; }
+
+ [JsonPropertyName("statusCode")]
+ public int? StatusCode { get; set; }
+
+ [JsonPropertyName("error")]
+ public string? Error { get; set; }
+
+ [JsonPropertyName("metadata")]
+ public object? Metadata { get; set; }
+
+ [JsonPropertyName("diff")]
+ public object? Diff { get; set; }
+
+ [JsonPropertyName("createdAt")]
+ public string? CreatedAt { get; set; }
+}
+
+public class MonitorCheckDetail : MonitorCheck
+{
+ [JsonPropertyName("pages")]
+ public List<MonitorCheckPage>? Pages { get; set; }
+
+ [JsonPropertyName("next")]
+ public string? Next { get; set; }
+}
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.ex
index d974d853b..3dce580e8 100644
--- a/apps/elixir-sdk/lib/firecrawl.ex
+++ b/apps/elixir-sdk/lib/firecrawl.ex
@@ -1157,4 +1157,274 @@ defmodule Firecrawl do
Req.delete!(client(opts), url: "/scrape/#{job_id}/interact")
end
+ @monitor_key_mapping %{
+ name: "name",
+ schedule: "schedule",
+ targets: "targets",
+ webhook: "webhook",
+ notification: "notification",
+ retention_days: "retentionDays",
+ status: "status"
+ }
+
+ @monitor_list_key_mapping %{limit: "limit", offset: "offset"}
+ @monitor_check_list_key_mapping %{limit: "limit", offset: "offset", status: "status"}
+ @monitor_check_key_mapping %{limit: "limit", skip: "skip", status: "status"}
+
+ @monitor_schema NimbleOptions.new!([
+ name: [type: :string],
+ schedule: [type: :any],
+ targets: [type: {:list, :any}],
+ webhook: [type: :any],
+ notification: [type: :any],
+ retention_days: [type: :integer],
+ status: [type: :any]
+ ])
+
+ @monitor_list_schema NimbleOptions.new!([
+ limit: [type: :integer],
+ offset: [type: :integer]
+ ])
+
+ @monitor_check_list_schema NimbleOptions.new!([
+ limit: [type: :integer],
+ offset: [type: :integer],
+ status: [type: :any]
+ ])
+
+ @monitor_check_schema NimbleOptions.new!([
+ limit: [type: :integer],
+ skip: [type: :integer],
+ status: [type: :any]
+ ])
+
+ @doc """
+ Create a scheduled monitor.
+
+ `POST /monitor`
+ """
+ @spec create_monitor(keyword(), keyword()) :: response()
+ def create_monitor(params \\ [], opts \\ []) do
+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_schema) do
+ Req.post(client(opts), url: "/monitor", json: to_body(params, @monitor_key_mapping))
+ end
+ end
+
+ @doc """
+ Bang variant of `create_monitor`. Raises on error.
+ """
+ @spec create_monitor!(keyword(), keyword()) :: Req.Response.t()
+ def create_monitor!(params \\ [], opts \\ []) do
+ params = NimbleOptions.validate!(params, @monitor_schema)
+ Req.post!(client(opts), url: "/monitor", json: to_body(params, @monitor_key_mapping))
+ end
+
+ @doc """
+ List monitors for the authenticated team.
+
+ `GET /monitor`
+ """
+ @spec list_monitors(keyword(), keyword()) :: response()
+ def list_monitors(params \\ [], opts \\ []) do
+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_list_schema) do
+ Req.get(client(opts), url: "/monitor", params: to_query(params, @monitor_list_key_mapping))
+ end
+ end
+
+ @doc """
+ Bang variant of `list_monitors`. Raises on error.
+ """
+ @spec list_monitors!(keyword(), keyword()) :: Req.Response.t()
+ def list_monitors!(params \\ [], opts \\ []) do
+ params = NimbleOptions.validate!(params, @monitor_list_schema)
+ Req.get!(client(opts), url: "/monitor", params: to_query(params, @monitor_list_key_mapping))
+ end
+
+ @doc """
+ Get a monitor by ID.
+
+ `GET /monitor/{monitorId}`
+ """
+ @spec get_monitor(String.t(), keyword()) :: response()
+ def get_monitor(monitor_id, opts \\ []) do
+ Req.get(client(opts), url: "/monitor/#{monitor_id}")
+ end
+
+ @doc """
+ Bang variant of `get_monitor`. Raises on error.
+ """
+ @spec get_monitor!(String.t(), keyword()) :: Req.Response.t()
+ def get_monitor!(monitor_id, opts \\ []) do
+ Req.get!(client(opts), url: "/monitor/#{monitor_id}")
+ end
+
+ @doc """
+ Update a monitor.
+
+ `PATCH /monitor/{monitorId}`
+ """
+ @spec update_monitor(String.t(), keyword(), keyword()) :: response()
+ def update_monitor(monitor_id, params \\ [], opts \\ []) do
+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_schema) do
+ Req.patch(client(opts), url: "/monitor/#{monitor_id}", json: to_body(params, @monitor_key_mapping))
+ end
+ end
+
+ @doc """
+ Bang variant of `update_monitor`. Raises on error.
+ """
+ @spec update_monitor!(String.t(), keyword(), keyword()) :: Req.Response.t()
+ def update_monitor!(monitor_id, params \\ [], opts \\ []) do
+ params = NimbleOptions.validate!(params, @monitor_schema)
+ Req.patch!(client(opts), url: "/monitor/#{monitor_id}", json: to_body(params, @monitor_key_mapping))
+ end
+
+ @doc """
+ Delete a monitor.
+
+ `DELETE /monitor/{monitorId}`
+ """
+ @spec delete_monitor(String.t(), keyword()) :: response()
+ def delete_monitor(monitor_id, opts \\ []) do
+ Req.delete(client(opts), url: "/monitor/#{monitor_id}")
+ end
+
+ @doc """
+ Bang variant of `delete_monitor`. Raises on error.
+ """
+ @spec delete_monitor!(String.t(), keyword()) :: Req.Response.t()
+ def delete_monitor!(monitor_id, opts \\ []) do
+ Req.delete!(client(opts), url: "/monitor/#{monitor_id}")
+ end
+
+ @doc """
+ Trigger a manual monitor check.
+
+ `POST /monitor/{monitorId}/run`
+ """
+ @spec run_monitor(String.t(), keyword()) :: response()
+ def run_monitor(monitor_id, opts \\ []) do
+ Req.post(client(opts), url: "/monitor/#{monitor_id}/run", json: %{})
+ end
+
+ @doc """
+ Bang variant of `run_monitor`. Raises on error.
+ """
+ @spec run_monitor!(String.t(), keyword()) :: Req.Response.t()
+ def run_monitor!(monitor_id, opts \\ []) do
+ Req.post!(client(opts), url: "/monitor/#{monitor_id}/run", json: %{})
+ end
+
+ @doc """
+ List checks for a monitor.
+
+ `GET /monitor/{monitorId}/checks`
+ """
+ @spec list_monitor_checks(String.t(), keyword(), keyword()) :: response()
+ def list_monitor_checks(monitor_id, params \\ [], opts \\ []) do
+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_check_list_schema) do
+ Req.get(client(opts),
+ url: "/monitor/#{monitor_id}/checks",
+ params: to_query(params, @monitor_check_list_key_mapping)
+ )
+ end
+ end
+
+ @doc """
+ Bang variant of `list_monitor_checks`. Raises on error.
+ """
+ @spec list_monitor_checks!(String.t(), keyword(), keyword()) :: Req.Response.t()
+ def list_monitor_checks!(monitor_id, params \\ [], opts \\ []) do
+ params = NimbleOptions.validate!(params, @monitor_check_list_schema)
+ Req.get!(client(opts),
+ url: "/monitor/#{monitor_id}/checks",
+ params: to_query(params, @monitor_check_list_key_mapping)
+ )
+ end
+
+ @doc """
+ Get a monitor check with paginated page results and inline diffs.
+
+ `GET /monitor/{monitorId}/checks/{checkId}`
+ """
+ @spec get_monitor_check(String.t(), String.t(), keyword(), keyword()) :: response()
+ def get_monitor_check(monitor_id, check_id, params \\ [], opts \\ []) do
+ {auto_paginate, opts} = Keyword.pop(opts, :auto_paginate, true)
+
+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_check_schema) do
+ case Req.get(client(opts),
+ url: "/monitor/#{monitor_id}/checks/#{check_id}",
+ params: to_query(params, @monitor_check_key_mapping)
+ ) do
+ {:ok, response} when auto_paginate -> {:ok, paginate_monitor_check_response(response, opts)}
+ result -> result
+ end
+ end
+ end
+
+ @doc """
+ Bang variant of `get_monitor_check`. Raises on error.
+ """
+ @spec get_monitor_check!(String.t(), String.t(), keyword(), keyword()) :: Req.Response.t()
+ def get_monitor_check!(monitor_id, check_id, params \\ [], opts \\ []) do
+ {auto_paginate, opts} = Keyword.pop(opts, :auto_paginate, true)
+ params = NimbleOptions.validate!(params, @monitor_check_schema)
+ response = Req.get!(client(opts),
+ url: "/monitor/#{monitor_id}/checks/#{check_id}",
+ params: to_query(params, @monitor_check_key_mapping)
+ )
+
+ if auto_paginate, do: paginate_monitor_check_response!(response, opts), else: response
+ end
+
+ defp paginate_monitor_check_response(%Req.Response{body: body} = response, opts) when is_map(body) do
+ data = Map.get(body, "data") || %{}
+ pages = Map.get(data, "pages") || []
+ next = Map.get(body, "next") || Map.get(data, "next")
+ pages = fetch_monitor_check_pages(next, pages, opts)
+ data = data |> Map.put("pages", pages) |> Map.put("next", nil)
+ %{response | body: body |> Map.put("data", data) |> Map.put("next", nil)}
+ end
+
+ defp paginate_monitor_check_response(response, _opts), do: response
+
+ defp paginate_monitor_check_response!(%Req.Response{body: body} = response, opts) when is_map(body) do
+ data = Map.get(body, "data") || %{}
+ pages = Map.get(data, "pages") || []
+ next = Map.get(body, "next") || Map.get(data, "next")
+ pages = fetch_monitor_check_pages!(next, pages, opts)
+ data = data |> Map.put("pages", pages) |> Map.put("next", nil)
+ %{response | body: body |> Map.put("data", data) |> Map.put("next", nil)}
+ end
+
+ defp paginate_monitor_check_response!(response, _opts), do: response
+
+ defp fetch_monitor_check_pages(nil, pages, _opts), do: pages
+ defp fetch_monitor_check_pages("", pages, _opts), do: pages
+
+ defp fetch_monitor_check_pages(next, pages, opts) do
+ case Req.get(client(opts), url: next) do
+ {:ok, %Req.Response{body: body}} when is_map(body) ->
+ data = Map.get(body, "data") || %{}
+ next_pages = Map.get(data, "pages") || []
+ next_url = Map.get(body, "next") || Map.get(data, "next")
+ fetch_monitor_check_pages(next_url, pages ++ next_pages, opts)
+
+ _ ->
+ pages
+ end
+ end
+
+ defp fetch_monitor_check_pages!(nil, pages, _opts), do: pages
+ defp fetch_monitor_check_pages!("", pages, _opts), do: pages
+
+ defp fetch_monitor_check_pages!(next, pages, opts) do
+ response = Req.get!(client(opts), url: next)
+ body = response.body
+ data = if is_map(body), do: Map.get(body, "data") || %{}, else: %{}
+ next_pages = Map.get(data, "pages") || []
+ next_url = if is_map(body), do: Map.get(body, "next") || Map.get(data, "next"), else: nil
+ fetch_monitor_check_pages!(next_url, pages ++ next_pages, opts)
+ end
+
end
diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exs
index 479cf2217..e15468af2 100644
--- a/apps/elixir-sdk/mix.exs
+++ b/apps/elixir-sdk/mix.exs
@@ -1,7 +1,7 @@
defmodule Firecrawl.MixProject do
use Mix.Project
- @version "1.2.1"
+ @version "1.3.0"
@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"
def project do
diff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.go
index 33f0361f7..57f608e0c 100644
--- a/apps/go-sdk/firecrawl.go
+++ b/apps/go-sdk/firecrawl.go
@@ -16,6 +16,7 @@ import (
"encoding/json"
"fmt"
"net/http"
+ "net/url"
"os"
"strings"
"time"
@@ -24,7 +25,7 @@ import (
)
const (
- defaultPollInterval = 2 // seconds
+ defaultPollInterval = 2 // seconds
defaultJobTimeout = 300 // seconds
)
@@ -383,6 +384,145 @@ func (c *Client) Map(ctx context.Context, url string, opts *MapOptions) (*MapDat
return data, nil
}
+// ================================================================
+// MONITOR
+// ================================================================
+
+// CreateMonitor creates a scheduled monitor.
+func (c *Client) CreateMonitor(ctx context.Context, req *MonitorCreateRequest) (*Monitor, error) {
+ if req == nil {
+ return nil, &FirecrawlError{Message: "monitor request is required"}
+ }
+ raw, err := c.http.post(ctx, "/v2/monitor", req, nil)
+ if err != nil {
+ return nil, err
+ }
+ return extractDataAs[Monitor](raw)
+}
+
+// ListMonitors lists monitors for the authenticated team.
+func (c *Client) ListMonitors(ctx context.Context, opts *ListMonitorsOptions) ([]Monitor, error) {
+ path := "/v2/monitor" + listQuery(opts)
+ raw, err := c.http.get(ctx, path)
+ if err != nil {
+ return nil, err
+ }
+ data, err := extractDataAs[[]Monitor](raw)
+ if err != nil {
+ return nil, err
+ }
+ return *data, nil
+}
+
+// GetMonitor gets a monitor by ID.
+func (c *Client) GetMonitor(ctx context.Context, monitorID string) (*Monitor, error) {
+ if monitorID == "" {
+ return nil, &FirecrawlError{Message: "monitor ID is required"}
+ }
+ raw, err := c.http.get(ctx, "/v2/monitor/"+monitorID)
+ if err != nil {
+ return nil, err
+ }
+ return extractDataAs[Monitor](raw)
+}
+
+// UpdateMonitor updates a monitor.
+func (c *Client) UpdateMonitor(ctx context.Context, monitorID string, req *MonitorUpdateRequest) (*Monitor, error) {
+ if monitorID == "" {
+ return nil, &FirecrawlError{Message: "monitor ID is required"}
+ }
+ if req == nil {
+ return nil, &FirecrawlError{Message: "monitor update request is required"}
+ }
+ raw, err := c.http.patch(ctx, "/v2/monitor/"+monitorID, req)
+ if err != nil {
+ return nil, err
+ }
+ return extractDataAs[Monitor](raw)
+}
+
+// DeleteMonitor deletes a monitor.
+func (c *Client) DeleteMonitor(ctx context.Context, monitorID string) (bool, error) {
+ if monitorID == "" {
+ return false, &FirecrawlError{Message: "monitor ID is required"}
+ }
+ raw, err := c.http.delete(ctx, "/v2/monitor/"+monitorID)
+ if err != nil {
+ return false, err
+ }
+ var resp struct {
+ Success bool `json:"success"`
+ }
+ if err := json.Unmarshal(raw, &resp); err != nil {
+ return false, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}
+ }
+ return resp.Success, nil
+}
+
+// RunMonitor triggers a manual monitor check.
+func (c *Client) RunMonitor(ctx context.Context, monitorID string) (*MonitorCheck, error) {
+ if monitorID == "" {
+ return nil, &FirecrawlError{Message: "monitor ID is required"}
+ }
+ raw, err := c.http.post(ctx, "/v2/monitor/"+monitorID+"/run", map[string]interface{}{}, nil)
+ if err != nil {
+ return nil, err
+ }
+ return extractDataAs[MonitorCheck](raw)
+}
+
+// ListMonitorChecks lists checks for a monitor.
+func (c *Client) ListMonitorChecks(ctx context.Context, monitorID string, opts *ListMonitorChecksOptions) ([]MonitorCheck, error) {
+ if monitorID == "" {
+ return nil, &FirecrawlError{Message: "monitor ID is required"}
+ }
+ path := "/v2/monitor/" + monitorID + "/checks" + checksQuery(opts)
+ raw, err := c.http.get(ctx, path)
+ if err != nil {
+ return nil, err
+ }
+ data, err := extractDataAs[[]MonitorCheck](raw)
+ if err != nil {
+ return nil, err
+ }
+ return *data, nil
+}
+
+// GetMonitorCheck gets a monitor check with page results, auto-paginated by default.
+func (c *Client) GetMonitorCheck(ctx context.Context, monitorID, checkID string, opts *GetMonitorCheckOptions) (*MonitorCheckDetail, error) {
+ if monitorID == "" || checkID == "" {
+ return nil, &FirecrawlError{Message: "monitor ID and check ID are required"}
+ }
+ path := "/v2/monitor/" + monitorID + "/checks/" + checkID + monitorCheckPageQuery(opts)
+ raw, err := c.http.get(ctx, path)
+ if err != nil {
+ return nil, err
+ }
+ detail, err := extractMonitorCheckDetail(raw)
+ if err != nil {
+ return nil, err
+ }
+
+ autoPaginate := true
+ if opts != nil && opts.AutoPaginate != nil {
+ autoPaginate = *opts.AutoPaginate
+ }
+ for autoPaginate && detail.Next != "" {
+ raw, err := c.http.getAbsolute(ctx, detail.Next)
+ if err != nil {
+ return nil, err
+ }
+ nextPage, err := extractMonitorCheckDetail(raw)
+ if err != nil {
+ return nil, err
+ }
+ detail.Pages = append(detail.Pages, nextPage.Pages...)
+ detail.Next = nextPage.Next
+ }
+
+ return detail, nil
+}
+
// ================================================================
// SEARCH
// ================================================================
@@ -793,6 +933,63 @@ func mergeOptions(body map[string]interface{}, opts interface{}) {
}
}
+func listQuery(opts *ListMonitorsOptions) string {
+ if opts == nil {
+ return ""
+ }
+ values := url.Values{}
+ if opts.Limit != nil {
+ values.Set("limit", fmt.Sprintf("%d", *opts.Limit))
+ }
+ if opts.Offset != nil {
+ values.Set("offset", fmt.Sprintf("%d", *opts.Offset))
+ }
+ if encoded := values.Encode(); encoded != "" {
+ return "?" + encoded
+ }
+ return ""
+}
+
+func checksQuery(opts *ListMonitorChecksOptions) string {
+ if opts == nil {
+ return ""
+ }
+ values := url.Values{}
+ if opts.Limit != nil {
+ values.Set("limit", fmt.Sprintf("%d", *opts.Limit))
+ }
+ if opts.Offset != nil {
+ values.Set("offset", fmt.Sprintf("%d", *opts.Offset))
+ }
+ if opts.Status != "" {
+ values.Set("status", opts.Status)
+ }
+ if encoded := values.Encode(); encoded != "" {
+ return "?" + encoded
+ }
+ return ""
+}
+
+func monitorCheckPageQuery(opts *GetMonitorCheckOptions) string {
+ if opts == nil {
+ return ""
+ }
+ values := url.Values{}
+ if opts.Limit != nil {
+ values.Set("limit", fmt.Sprintf("%d", *opts.Limit))
+ }
+ if opts.Skip != nil {
+ values.Set("skip", fmt.Sprintf("%d", *opts.Skip))
+ }
+ if opts.Status != "" {
+ values.Set("status", opts.Status)
+ }
+ if encoded := values.Encode(); encoded != "" {
+ return "?" + encoded
+ }
+ return ""
+}
+
// extractDataAs extracts the "data" field from a raw API response and deserializes it.
func extractDataAs[T any](raw json.RawMessage) (*T, error) {
var envelope map[string]json.RawMessage
@@ -816,3 +1013,17 @@ func extractDataAs[T any](raw json.RawMessage) (*T, error) {
}
return &result, nil
}
+
+func extractMonitorCheckDetail(raw json.RawMessage) (*MonitorCheckDetail, error) {
+ var envelope struct {
+ Data MonitorCheckDetail `json:"data"`
+ Next string `json:"next"`
+ }
+ if err := json.Unmarshal(raw, &envelope); err != nil {
+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}
+ }
+ if envelope.Next != "" {
+ envelope.Data.Next = envelope.Next
+ }
+ return &envelope.Data, nil
+}
diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.go
index 99156559b..e17f221d6 100644
--- a/apps/go-sdk/http_client.go
+++ b/apps/go-sdk/http_client.go
@@ -49,6 +49,12 @@ func (h *httpClient) post(ctx context.Context, path string, body interface{}, ex
return h.doJSON(ctx, "POST", url, body, extraHeaders)
}
+// patch sends a PATCH request.
+func (h *httpClient) patch(ctx context.Context, path string, body interface{}) (json.RawMessage, error) {
+ url := h.baseURL + path
+ return h.doJSON(ctx, "PATCH", url, body, nil)
+}
+
// get sends a GET request.
func (h *httpClient) get(ctx context.Context, path string) (json.RawMessage, error) {
url := h.baseURL + path
diff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.go
index d4335d74e..5877eadea 100644
--- a/apps/go-sdk/models.go
+++ b/apps/go-sdk/models.go
@@ -100,6 +100,126 @@ type MapData struct {
Links []LinkResult `json:"links,omitempty"`
}
+// MonitorSchedule configures when a monitor runs.
+type MonitorSchedule struct {
+ Cron string `json:"cron"`
+ Timezone string `json:"timezone,omitempty"`
+}
+
+// MonitorCreateRequest creates a scheduled monitor.
+type MonitorCreateRequest struct {
+ Name string `json:"name"`
+ Schedule MonitorSchedule `json:"schedule"`
+ Targets []map[string]interface{} `json:"targets"`
+ Webhook map[string]interface{} `json:"webhook,omitempty"`
+ Notification map[string]interface{} `json:"notification,omitempty"`
+ RetentionDays *int `json:"retentionDays,omitempty"`
+}
+
+// MonitorUpdateRequest updates a scheduled monitor.
+type MonitorUpdateRequest struct {
+ Name string `json:"name,omitempty"`
+ Status string `json:"status,omitempty"`
+ Schedule *MonitorSchedule `json:"schedule,omitempty"`
+ Targets []map[string]interface{} `json:"targets,omitempty"`
+ Webhook map[string]interface{} `json:"webhook,omitempty"`
+ Notification map[string]interface{} `json:"notification,omitempty"`
+ RetentionDays *int `json:"retentionDays,omitempty"`
+}
+
+// Monitor represents a scheduled monitor.
+type Monitor struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Status string `json:"status"`
+ Schedule MonitorSchedule `json:"schedule"`
+ NextRunAt string `json:"nextRunAt,omitempty"`
+ LastRunAt string `json:"lastRunAt,omitempty"`
+ CurrentCheckID string `json:"currentCheckId,omitempty"`
+ Targets []map[string]interface{} `json:"targets,omitempty"`
+ Webhook map[string]interface{} `json:"webhook,omitempty"`
+ Notification map[string]interface{} `json:"notification,omitempty"`
+ RetentionDays int `json:"retentionDays"`
+ EstimatedCreditsPerMonth *int `json:"estimatedCreditsPerMonth,omitempty"`
+ LastCheckSummary *MonitorSummary `json:"lastCheckSummary,omitempty"`
+ CreatedAt string `json:"createdAt,omitempty"`
+ UpdatedAt string `json:"updatedAt,omitempty"`
+}
+
+// MonitorSummary summarizes page statuses in a check.
+type MonitorSummary struct {
+ TotalPages int `json:"totalPages"`
+ Same int `json:"same"`
+ Changed int `json:"changed"`
+ New int `json:"new"`
+ Removed int `json:"removed"`
+ Error int `json:"error"`
+}
+
+// MonitorCheck represents a single monitor run.
+type MonitorCheck struct {
+ ID string `json:"id"`
+ MonitorID string `json:"monitorId"`
+ Status string `json:"status"`
+ Trigger string `json:"trigger"`
+ ScheduledFor string `json:"scheduledFor,omitempty"`
+ StartedAt string `json:"startedAt,omitempty"`
+ FinishedAt string `json:"finishedAt,omitempty"`
+ EstimatedCredits *int `json:"estimatedCredits,omitempty"`
+ ReservedCredits *int `json:"reservedCredits,omitempty"`
+ ActualCredits *int `json:"actualCredits,omitempty"`
+ BillingStatus string `json:"billingStatus,omitempty"`
+ Summary MonitorSummary `json:"summary"`
+ TargetResults interface{} `json:"targetResults,omitempty"`
+ NotificationStatus interface{} `json:"notificationStatus,omitempty"`
+ Error string `json:"error,omitempty"`
+ CreatedAt string `json:"createdAt,omitempty"`
+ UpdatedAt string `json:"updatedAt,omitempty"`
+}
+
+// MonitorCheckPage is a single page result in a monitor check.
+type MonitorCheckPage struct {
+ ID string `json:"id"`
+ TargetID string `json:"targetId"`
+ URL string `json:"url"`
+ Status string `json:"status"`
+ PreviousScrapeID string `json:"previousScrapeId,omitempty"`
+ CurrentScrapeID string `json:"currentScrapeId,omitempty"`
+ StatusCode *int `json:"statusCode,omitempty"`
+ Error string `json:"error,omitempty"`
+ Metadata interface{} `json:"metadata,omitempty"`
+ Diff interface{} `json:"diff,omitempty"`
+ CreatedAt string `json:"createdAt,omitempty"`
+}
+
+// MonitorCheckDetail includes paginated page results and inline diffs.
+type MonitorCheckDetail struct {
+ MonitorCheck
+ Pages []MonitorCheckPage `json:"pages,omitempty"`
+ Next string `json:"next,omitempty"`
+}
+
+// ListMonitorsOptions controls monitor list pagination.
+type ListMonitorsOptions struct {
+ Limit *int
+ Offset *int
+}
+
+// ListMonitorChecksOptions controls monitor check pagination/filtering.
+type ListMonitorChecksOptions struct {
+ Limit *int
+ Offset *int
+ Status string
+}
+
+// GetMonitorCheckOptions controls monitor check page pagination/filtering.
+type GetMonitorCheckOptions struct {
+ Limit *int
+ Skip *int
+ Status string
+ AutoPaginate *bool
+}
+
// SearchData represents the result of a search request.
type SearchData struct {
Web []map[string]interface{} `json:"web,omitempty"`
@@ -142,13 +262,13 @@ type BrowserCreateResponse struct {
// BrowserExecuteResponse is returned when executing code in a browser session.
type BrowserExecuteResponse struct {
- Success bool `json:"success"`
- Stdout string `json:"stdout,omitempty"`
- Result string `json:"result,omitempty"`
- Stderr string `json:"stderr,omitempty"`
- ExitCode *int `json:"exitCode,omitempty"`
- Killed *bool `json:"killed,omitempty"`
- Error string `json:"error,omitempty"`
+ Success bool `json:"success"`
+ Stdout string `json:"stdout,omitempty"`
+ Result string `json:"result,omitempty"`
+ Stderr string `json:"stderr,omitempty"`
+ ExitCode *int `json:"exitCode,omitempty"`
+ Killed *bool `json:"killed,omitempty"`
+ Error string `json:"error,omitempty"`
}
// BrowserDeleteResponse is returned when deleting a browser session.
diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.kts
index 3183e7110..3479bb1e7 100644
--- a/apps/java-sdk/build.gradle.kts
+++ b/apps/java-sdk/build.gradle.kts
@@ -4,7 +4,7 @@ plugins {
}
group = "com.firecrawl"
-version = "1.4.1"
+version = "1.5.0"
java {
sourceCompatibility = JavaVersion.VERSION_11
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
index 9055be3e7..bfca8dfc0 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java
@@ -446,6 +446,81 @@ public class FirecrawlClient {
return extractData(http.post("/v2/map", body, Map.class), MapData.class);
}
+ // ================================================================
+ // MONITOR
+ // ================================================================
+
+ public Monitor createMonitor(Map<String, Object> request) {
+ Objects.requireNonNull(request, "Monitor request is required");
+ return extractData(http.post("/v2/monitor", request, Map.class), Monitor.class);
+ }
+
+ public List<Monitor> listMonitors() {
+ return listMonitors(null, null);
+ }
+
+ public List<Monitor> listMonitors(Integer limit, Integer offset) {
+ Map raw = http.get("/v2/monitor" + listQuery(limit, offset), Map.class);
+ return extractDataList(raw, Monitor.class);
+ }
+
+ public Monitor getMonitor(String monitorId) {
+ Objects.requireNonNull(monitorId, "Monitor ID is required");
+ return extractData(http.get("/v2/monitor/" + monitorId, Map.class), Monitor.class);
+ }
+
+ public Monitor updateMonitor(String monitorId, Map<String, Object> request) {
+ Objects.requireNonNull(monitorId, "Monitor ID is required");
+ Objects.requireNonNull(request, "Monitor update request is required");
+ return extractData(http.patch("/v2/monitor/" + monitorId, request, Map.class), Monitor.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ public boolean deleteMonitor(String monitorId) {
+ Objects.requireNonNull(monitorId, "Monitor ID is required");
+ Map<String, Object> response = http.delete("/v2/monitor/" + monitorId, Map.class);
+ return Boolean.TRUE.equals(response.get("success"));
+ }
+
+ public MonitorCheck runMonitor(String monitorId) {
+ Objects.requireNonNull(monitorId, "Monitor ID is required");
+ return extractData(
+ http.post("/v2/monitor/" + monitorId + "/run", Collections.emptyMap(), Map.class),
+ MonitorCheck.class
+ );
+ }
+
+ public List<MonitorCheck> listMonitorChecks(String monitorId) {
+ return listMonitorChecks(monitorId, null, null);
+ }
+
+ public List<MonitorCheck> listMonitorChecks(String monitorId, Integer limit, Integer offset) {
+ Objects.requireNonNull(monitorId, "Monitor ID is required");
+ Map raw = http.get("/v2/monitor/" + monitorId + "/checks" + listQuery(limit, offset), Map.class);
+ return extractDataList(raw, MonitorCheck.class);
+ }
+
+ public MonitorCheckDetail getMonitorCheck(String monitorId, String checkId) {
+ return getMonitorCheck(monitorId, checkId, null, null, null, true);
+ }
+
+ public MonitorCheckDetail getMonitorCheck(
+ String monitorId, String checkId, Integer limit, Integer skip, String status) {
+ return getMonitorCheck(monitorId, checkId, limit, skip, status, true);
+ }
+
+ public MonitorCheckDetail getMonitorCheck(
+ String monitorId, String checkId, Integer limit, Integer skip, String status, boolean autoPaginate) {
+ Objects.requireNonNull(monitorId, "Monitor ID is required");
+ Objects.requireNonNull(checkId, "Check ID is required");
+ MonitorCheckDetail check = extractData(
+ http.get("/v2/monitor/" + monitorId + "/checks/" + checkId
+ + monitorCheckQuery(limit, skip, status), Map.class),
+ MonitorCheckDetail.class
+ );
+ return autoPaginate ? paginateMonitorCheck(check) : check;
+ }
+
// ================================================================
// SEARCH
// ================================================================
@@ -845,6 +920,50 @@ public class FirecrawlClient {
return CompletableFuture.supplyAsync(() -> map(url, options), asyncExecutor);
}
+ public CompletableFuture<Monitor> createMonitorAsync(Map<String, Object> request) {
+ return CompletableFuture.supplyAsync(() -> createMonitor(request), asyncExecutor);
+ }
+
+ public CompletableFuture<List<Monitor>> listMonitorsAsync(Integer limit, Integer offset) {
+ return CompletableFuture.supplyAsync(() -> listMonitors(limit, offset), asyncExecutor);
+ }
+
+ public CompletableFuture<Monitor> getMonitorAsync(String monitorId) {
+ return CompletableFuture.supplyAsync(() -> getMonitor(monitorId), asyncExecutor);
+ }
+
+ public CompletableFuture<Monitor> updateMonitorAsync(String monitorId, Map<String, Object> request) {
+ return CompletableFuture.supplyAsync(() -> updateMonitor(monitorId, request), asyncExecutor);
+ }
+
+ public CompletableFuture<Boolean> deleteMonitorAsync(String monitorId) {
+ return CompletableFuture.supplyAsync(() -> deleteMonitor(monitorId), asyncExecutor);
+ }
+
+ public CompletableFuture<MonitorCheck> runMonitorAsync(String monitorId) {
+ return CompletableFuture.supplyAsync(() -> runMonitor(monitorId), asyncExecutor);
+ }
+
+ public CompletableFuture<List<MonitorCheck>> listMonitorChecksAsync(String monitorId, Integer limit, Integer offset) {
+ return CompletableFuture.supplyAsync(() -> listMonitorChecks(monitorId, limit, offset), asyncExecutor);
+ }
+
+ public CompletableFuture<MonitorCheckDetail> getMonitorCheckAsync(
+ String monitorId, String checkId, Integer limit, Integer skip, String status) {
+ return CompletableFuture.supplyAsync(
+ () -> getMonitorCheck(monitorId, checkId, limit, skip, status),
+ asyncExecutor
+ );
+ }
+
+ public CompletableFuture<MonitorCheckDetail> getMonitorCheckAsync(
+ String monitorId, String checkId, Integer limit, Integer skip, String status, boolean autoPaginate) {
+ return CompletableFuture.supplyAsync(
+ () -> getMonitorCheck(monitorId, checkId, limit, skip, status, autoPaginate),
+ asyncExecutor
+ );
+ }
+
/**
* Asynchronously runs an agent task and waits for completion.
*
@@ -1076,6 +1195,25 @@ public class FirecrawlClient {
return job;
}
+ private MonitorCheckDetail paginateMonitorCheck(MonitorCheckDetail check) {
+ if (check.getPages() == null) {
+ check.setPages(new ArrayList<>());
+ }
+ MonitorCheckDetail current = check;
+ while (current.getNext() != null && !current.getNext().isEmpty()) {
+ MonitorCheckDetail nextPage = extractData(
+ http.getAbsolute(current.getNext(), Map.class),
+ MonitorCheckDetail.class
+ );
+ if (nextPage.getPages() != null && !nextPage.getPages().isEmpty()) {
+ check.getPages().addAll(nextPage.getPages());
+ }
+ current = nextPage;
+ }
+ check.setNext(null);
+ return check;
+ }
+
// ================================================================
// INTERNAL UTILITIES
// ================================================================
@@ -1093,6 +1231,35 @@ public class FirecrawlClient {
return http.objectMapper.convertValue(data, type);
}
+ private <T> List<T> extractDataList(Map rawResponse, Class<T> type) {
+ Object data = rawResponse.get("data");
+ if (!(data instanceof List<?>)) {
+ return Collections.emptyList();
+ }
+ List<T> result = new ArrayList<>();
+ for (Object item : (List<?>) data) {
+ result.add(http.objectMapper.convertValue(item, type));
+ }
+ return result;
+ }
+
+ private String listQuery(Integer limit, Integer offset) {
+ List<String> parts = new ArrayList<>();
+ if (limit != null) parts.add("limit=" + limit);
+ if (offset != null) parts.add("offset=" + offset);
+ return parts.isEmpty() ? "" : "?" + String.join("&", parts);
+ }
+
+ private String monitorCheckQuery(Integer limit, Integer skip, String status) {
+ List<String> parts = new ArrayList<>();
+ if (limit != null) parts.add("limit=" + limit);
+ if (skip != null) parts.add("skip=" + skip);
+ if (status != null && !status.isBlank()) {
+ parts.add("status=" + java.net.URLEncoder.encode(status, java.nio.charset.StandardCharsets.UTF_8));
+ }
+ return parts.isEmpty() ? "" : "?" + String.join("&", parts);
+ }
+
/**
* Merges a typed options object into a request body map, using Jackson serialization.
*/
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
index 33e1fe5d4..695c9e079 100644
--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java
@@ -86,6 +86,27 @@ class FirecrawlHttpClient {
return executeWithRetry(request, responseType);
}
+ /**
+ * Sends a PATCH request with JSON body.
+ */
+ <T> T patch(String path, Object body, Class<T> responseType) {
+ String url = baseUrl + path;
+ String json;
+ try {
+ json = objectMapper.writeValueAsString(body);
+ } catch (JsonProcessingException e) {
+ throw new FirecrawlException("Failed to serialize request body", e);
+ }
+ RequestBody requestBody = RequestBody.create(json, JSON);
+ Request request = new Request.Builder()
+ .url(url)
+ .header("Authorization", "Bearer " + apiKey)
+ .header("Content-Type", "application/json")
+ .patch(requestBody)
+ .build();
+ return executeWithRetry(request, responseType);
+ }
+
/**
* Sends a POST multipart/form-data request.
*/
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Monitor.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Monitor.java
new file mode 100644
index 000000000..c881fb9f9
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Monitor.java
@@ -0,0 +1,40 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import java.util.List;
+import java.util.Map;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class Monitor {
+ private String id;
+ private String name;
+ private String status;
+ private MonitorSchedule schedule;
+ private String nextRunAt;
+ private String lastRunAt;
+ private String currentCheckId;
+ private List<Map<String, Object>> targets;
+ private Map<String, Object> webhook;
+ private Map<String, Object> notification;
+ private int retentionDays;
+ private Integer estimatedCreditsPerMonth;
+ private MonitorSummary lastCheckSummary;
+ private String createdAt;
+ private String updatedAt;
+
+ public String getId() { return id; }
+ public String getName() { return name; }
+ public String getStatus() { return status; }
+ public MonitorSchedule getSchedule() { return schedule; }
+ public String getNextRunAt() { return nextRunAt; }
+ public String getLastRunAt() { return lastRunAt; }
+ public String getCurrentCheckId() { return currentCheckId; }
+ public List<Map<String, Object>> getTargets() { return targets; }
+ public Map<String, Object> getWebhook() { return webhook; }
+ public Map<String, Object> getNotification() { return notification; }
+ public int getRetentionDays() { return retentionDays; }
+ public Integer getEstimatedCreditsPerMonth() { return estimatedCreditsPerMonth; }
+ public MonitorSummary getLastCheckSummary() { return lastCheckSummary; }
+ public String getCreatedAt() { return createdAt; }
+ public String getUpdatedAt() { return updatedAt; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheck.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheck.java
new file mode 100644
index 000000000..4b3f8212e
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheck.java
@@ -0,0 +1,42 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorCheck {
+ private String id;
+ private String monitorId;
+ private String status;
+ private String trigger;
+ private String scheduledFor;
+ private String startedAt;
+ private String finishedAt;
+ private Integer estimatedCredits;
+ private Integer reservedCredits;
+ private Integer actualCredits;
+ private String billingStatus;
+ private MonitorSummary summary;
+ private Object targetResults;
+ private Object notificationStatus;
+ private String error;
+ private String createdAt;
+ private String updatedAt;
+
+ public String getId() { return id; }
+ public String getMonitorId() { return monitorId; }
+ public String getStatus() { return status; }
+ public String getTrigger() { return trigger; }
+ public String getScheduledFor() { return scheduledFor; }
+ public String getStartedAt() { return startedAt; }
+ public String getFinishedAt() { return finishedAt; }
+ public Integer getEstimatedCredits() { return estimatedCredits; }
+ public Integer getReservedCredits() { return reservedCredits; }
+ public Integer getActualCredits() { return actualCredits; }
+ public String getBillingStatus() { return billingStatus; }
+ public MonitorSummary getSummary() { return summary; }
+ public Object getTargetResults() { return targetResults; }
+ public Object getNotificationStatus() { return notificationStatus; }
+ public String getError() { return error; }
+ public String getCreatedAt() { return createdAt; }
+ public String getUpdatedAt() { return updatedAt; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckDetail.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckDetail.java
new file mode 100644
index 000000000..cdb3c710d
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckDetail.java
@@ -0,0 +1,15 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import java.util.List;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorCheckDetail extends MonitorCheck {
+ private List<MonitorCheckPage> pages;
+ private String next;
+
+ public List<MonitorCheckPage> getPages() { return pages; }
+ public void setPages(List<MonitorCheckPage> pages) { this.pages = pages; }
+ public String getNext() { return next; }
+ public void setNext(String next) { this.next = next; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java
new file mode 100644
index 000000000..231d3788e
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java
@@ -0,0 +1,30 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorCheckPage {
+ private String id;
+ private String targetId;
+ private String url;
+ private String status;
+ private String previousScrapeId;
+ private String currentScrapeId;
+ private Integer statusCode;
+ private String error;
+ private Object metadata;
+ private Object diff;
+ private String createdAt;
+
+ public String getId() { return id; }
+ public String getTargetId() { return targetId; }
+ public String getUrl() { return url; }
+ public String getStatus() { return status; }
+ public String getPreviousScrapeId() { return previousScrapeId; }
+ public String getCurrentScrapeId() { return currentScrapeId; }
+ public Integer getStatusCode() { return statusCode; }
+ public String getError() { return error; }
+ public Object getMetadata() { return metadata; }
+ public Object getDiff() { return diff; }
+ public String getCreatedAt() { return createdAt; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSchedule.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSchedule.java
new file mode 100644
index 000000000..b8e3ff19f
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSchedule.java
@@ -0,0 +1,12 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorSchedule {
+ private String cron;
+ private String timezone;
+
+ public String getCron() { return cron; }
+ public String getTimezone() { return timezone; }
+}
diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSummary.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSummary.java
new file mode 100644
index 000000000..3f04548fc
--- /dev/null
+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSummary.java
@@ -0,0 +1,25 @@
+package com.firecrawl.models;
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MonitorSummary {
+ private int totalPages;
+ private int same;
+ private int changed;
+ private int newCount;
+ private int removed;
+ private int error;
+
+ public int getTotalPages() { return totalPages; }
+ public int getSame() { return same; }
+ public int getChanged() { return changed; }
+ public int getNew() { return newCount; }
+ public int getRemoved() { return removed; }
+ public int getError() { return error; }
+
+ @com.fasterxml.jackson.annotation.JsonProperty("new")
+ private void setNewCount(int value) {
+ this.newCount = value;
+ }
+}
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 51f8f4ace..f88ddd72b 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.21.1",
+ "version": "4.22.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts
index 20e2afe42..32e823197 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts
@@ -1,6 +1,7 @@
import { describe, test, expect, jest } from "@jest/globals";
import { getCrawlStatus } from "../../../v2/methods/crawl";
import { getBatchScrapeStatus } from "../../../v2/methods/batch";
+import { getMonitorCheck } from "../../../v2/methods/monitor";
describe("JS SDK v2 pagination", () => {
function makeHttp(getImpl: (url: string) => any) {
@@ -64,6 +65,26 @@ describe("JS SDK v2 pagination", () => {
expect(res.next).toBe("https://api/nextBatch");
});
+ test("monitor check: default autoPaginate aggregates pages and nulls next", async () => {
+ const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
+ const second = { status: 200, data: { success: true, next: null, data: { pages: [{ url: "b", status: "same" }], next: null } } };
+ const http = makeHttp((url) => {
+ if (url.includes("/v2/monitor/")) return first;
+ return second;
+ });
+ const res = await getMonitorCheck(http, "mon1", "check1");
+ expect(res.pages.length).toBe(2);
+ expect(res.next).toBeNull();
+ });
+
+ test("monitor check: autoPaginate=false returns next", async () => {
+ const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };
+ const http = makeHttp(() => first);
+ const res = await getMonitorCheck(http, "mon1", "check1", { autoPaginate: false });
+ expect(res.pages.length).toBe(1);
+ expect(res.next).toBe("https://api/m1");
+ });
+
test("crawl: maxWaitTime stops pagination after first page", async () => {
const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/n1", data: [{ markdown: "a" }] } };
const p1 = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 613a28f7f..f375bb457 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -32,6 +32,16 @@ import {
listBrowsers,
} from "./methods/browser";
import { getConcurrency, getCreditUsage, getQueueStatus, getTokenUsage, getCreditUsageHistorical, getTokenUsageHistorical } from "./methods/usage";
+import {
+ createMonitor as createMonitorMethod,
+ deleteMonitor as deleteMonitorMethod,
+ getMonitor as getMonitorMethod,
+ getMonitorCheck as getMonitorCheckMethod,
+ listMonitorChecks as listMonitorChecksMethod,
+ listMonitors as listMonitorsMethod,
+ runMonitor as runMonitorMethod,
+ updateMonitor as updateMonitorMethod,
+} from "./methods/monitor";
import type {
Document,
ParseFile,
@@ -60,6 +70,14 @@ import type {
ScrapeExecuteRequest,
ScrapeExecuteResponse,
ScrapeBrowserDeleteResponse,
+ CreateMonitorRequest,
+ ListMonitorChecksOptions,
+ ListMonitorsOptions,
+ Monitor,
+ MonitorCheck,
+ MonitorCheckDetail,
+ GetMonitorCheckOptions,
+ UpdateMonitorRequest,
} from "./types";
import { Watcher } from "./watcher";
import type { WatcherOptions } from "./watcher";
@@ -276,6 +294,73 @@ export class FirecrawlClient {
return crawlParamsPreview(this.http, url, prompt);
}
+ // Monitor
+ /**
+ * Create a scheduled monitor.
+ */
+ async createMonitor(request: CreateMonitorRequest): Promise<Monitor> {
+ return createMonitorMethod(this.http, request);
+ }
+
+ /**
+ * List monitors for the authenticated team.
+ */
+ async listMonitors(options?: ListMonitorsOptions): Promise<Monitor[]> {
+ return listMonitorsMethod(this.http, options);
+ }
+
+ /**
+ * Get a monitor by id.
+ */
+ async getMonitor(monitorId: string): Promise<Monitor> {
+ return getMonitorMethod(this.http, monitorId);
+ }
+
+ /**
+ * Update a monitor.
+ */
+ async updateMonitor(
+ monitorId: string,
+ request: UpdateMonitorRequest,
+ ): Promise<Monitor> {
+ return updateMonitorMethod(this.http, monitorId, request);
+ }
+
+ /**
+ * Delete a monitor.
+ */
+ async deleteMonitor(monitorId: string): Promise<boolean> {
+ return deleteMonitorMethod(this.http, monitorId);
+ }
+
+ /**
+ * Trigger a manual monitor check.
+ */
+ async runMonitor(monitorId: string): Promise<MonitorCheck> {
+ return runMonitorMethod(this.http, monitorId);
+ }
+
+ /**
+ * List checks for a monitor.
+ */
+ async listMonitorChecks(
+ monitorId: string,
+ options?: ListMonitorChecksOptions,
+ ): Promise<MonitorCheck[]> {
+ return listMonitorChecksMethod(this.http, monitorId, options);
+ }
+
+ /**
+ * Get a monitor check with paginated page results and inline diffs.
+ */
+ async getMonitorCheck(
+ monitorId: string,
+ checkId: string,
+ options?: GetMonitorCheckOptions,
+ ): Promise<MonitorCheckDetail> {
+ return getMonitorCheckMethod(this.http, monitorId, checkId, options);
+ }
+
// Batch
/**
* Start a batch scrape job for multiple URLs (async).
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/monitor.ts b/apps/js-sdk/firecrawl/src/v2/methods/monitor.ts
new file mode 100644
index 000000000..3e4221da8
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/v2/methods/monitor.ts
@@ -0,0 +1,181 @@
+import {
+ type CreateMonitorRequest,
+ type ListMonitorsOptions,
+ type ListMonitorChecksOptions,
+ type Monitor,
+ type MonitorCheck,
+ type MonitorCheckDetail,
+ type MonitorCheckPage,
+ type GetMonitorCheckOptions,
+ type UpdateMonitorRequest,
+} from "../types";
+import { HttpClient } from "../utils/httpClient";
+import {
+ throwForBadResponse,
+ normalizeAxiosError,
+} from "../utils/errorHandler";
+import { fetchAllPages } from "../utils/pagination";
+
+type ApiResponse<T> = {
+ success: boolean;
+ data?: T;
+ id?: string;
+ error?: string;
+};
+
+function queryString(params?: Record<string, unknown>): string {
+ if (!params) return "";
+ const query = new URLSearchParams();
+ for (const [key, value] of Object.entries(params)) {
+ if (value !== undefined && value !== null) query.set(key, String(value));
+ }
+ const str = query.toString();
+ return str ? `?${str}` : "";
+}
+
+function dataOrThrow<T>(res: { status: number; data?: ApiResponse<T> }, action: string): T {
+ if (res.status !== 200 || !res.data?.success || res.data.data == null) {
+ throwForBadResponse(res as any, action);
+ }
+ return res.data.data;
+}
+
+export async function createMonitor(
+ http: HttpClient,
+ request: CreateMonitorRequest,
+): Promise<Monitor> {
+ try {
+ const res = await http.post<ApiResponse<Monitor>>("/v2/monitor", request as any);
+ return dataOrThrow(res, "create monitor");
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "create monitor");
+ throw err;
+ }
+}
+
+export async function listMonitors(
+ http: HttpClient,
+ options?: ListMonitorsOptions,
+): Promise<Monitor[]> {
+ try {
+ const res = await http.get<ApiResponse<Monitor[]>>(
+ `/v2/monitor${queryString(options as Record<string, unknown>)}`,
+ );
+ return dataOrThrow(res, "list monitors");
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "list monitors");
+ throw err;
+ }
+}
+
+export async function getMonitor(
+ http: HttpClient,
+ monitorId: string,
+): Promise<Monitor> {
+ try {
+ const res = await http.get<ApiResponse<Monitor>>(`/v2/monitor/${monitorId}`);
+ return dataOrThrow(res, "get monitor");
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get monitor");
+ throw err;
+ }
+}
+
+export async function updateMonitor(
+ http: HttpClient,
+ monitorId: string,
+ request: UpdateMonitorRequest,
+): Promise<Monitor> {
+ try {
+ const res = await http.patch<ApiResponse<Monitor>>(
+ `/v2/monitor/${monitorId}`,
+ request as any,
+ );
+ return dataOrThrow(res, "update monitor");
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "update monitor");
+ throw err;
+ }
+}
+
+export async function deleteMonitor(
+ http: HttpClient,
+ monitorId: string,
+): Promise<boolean> {
+ try {
+ const res = await http.delete<ApiResponse<unknown>>(`/v2/monitor/${monitorId}`);
+ if (res.status !== 200 || !res.data?.success) {
+ throwForBadResponse(res, "delete monitor");
+ }
+ return true;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "delete monitor");
+ throw err;
+ }
+}
+
+export async function runMonitor(
+ http: HttpClient,
+ monitorId: string,
+): Promise<MonitorCheck> {
+ try {
+ const res = await http.post<ApiResponse<MonitorCheck>>(
+ `/v2/monitor/${monitorId}/run`,
+ {},
+ );
+ return dataOrThrow(res, "run monitor");
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "run monitor");
+ throw err;
+ }
+}
+
+export async function listMonitorChecks(
+ http: HttpClient,
+ monitorId: string,
+ options?: ListMonitorChecksOptions,
+): Promise<MonitorCheck[]> {
+ try {
+ const res = await http.get<ApiResponse<MonitorCheck[]>>(
+ `/v2/monitor/${monitorId}/checks${queryString(options as Record<string, unknown>)}`,
+ );
+ return dataOrThrow(res, "list monitor checks");
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "list monitor checks");
+ throw err;
+ }
+}
+
+export async function getMonitorCheck(
+ http: HttpClient,
+ monitorId: string,
+ checkId: string,
+ options?: GetMonitorCheckOptions,
+): Promise<MonitorCheckDetail> {
+ try {
+ const { autoPaginate: _autoPaginate, maxPages: _maxPages, maxResults: _maxResults, maxWaitTime: _maxWaitTime, ...query } = options ?? {};
+ const res = await http.get<ApiResponse<MonitorCheckDetail>>(
+ `/v2/monitor/${monitorId}/checks/${checkId}${queryString(query as Record<string, unknown>)}`,
+ );
+ const detail = dataOrThrow(res, "get monitor check");
+ const next = res.data?.next ?? detail.next ?? null;
+ const auto = options?.autoPaginate ?? true;
+ if (!auto || !next) {
+ return { ...detail, next };
+ }
+
+ return {
+ ...detail,
+ pages: await fetchAllPages<MonitorCheckPage>(
+ http,
+ next,
+ detail.pages || [],
+ options,
+ ),
+ next: null,
+ };
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get monitor check");
+ throw err;
+ }
+}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index f72a6b8ea..a07232dc9 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -606,6 +606,154 @@ export interface MapOptions {
location?: LocationConfig;
}
+export interface MonitorSchedule {
+ cron: string;
+ timezone?: string;
+}
+
+export interface MonitorEmailNotification {
+ enabled?: boolean;
+ recipients?: string[];
+ includeDiffs?: boolean;
+}
+
+export interface MonitorNotification {
+ email?: MonitorEmailNotification;
+}
+
+export interface MonitorWebhookConfig {
+ url: string;
+ headers?: Record<string, string>;
+ metadata?: Record<string, string>;
+ events?: string[];
+}
+
+export interface MonitorScrapeTarget {
+ id?: string;
+ type: 'scrape';
+ urls: string[];
+ scrapeOptions?: ScrapeOptions;
+}
+
+export interface MonitorCrawlTarget {
+ id?: string;
+ type: 'crawl';
+ url: string;
+ crawlOptions?: CrawlOptions;
+ scrapeOptions?: ScrapeOptions;
+}
+
+export type MonitorTarget = MonitorScrapeTarget | MonitorCrawlTarget;
+
+export interface CreateMonitorRequest {
+ name: string;
+ schedule: MonitorSchedule;
+ webhook?: MonitorWebhookConfig;
+ notification?: MonitorNotification;
+ targets: MonitorTarget[];
+ retentionDays?: number;
+}
+
+export interface UpdateMonitorRequest {
+ name?: string;
+ status?: 'active' | 'paused';
+ schedule?: MonitorSchedule;
+ webhook?: MonitorWebhookConfig | null;
+ notification?: MonitorNotification | null;
+ targets?: MonitorTarget[];
+ retentionDays?: number;
+}
+
+export interface MonitorSummary {
+ totalPages: number;
+ same: number;
+ changed: number;
+ new: number;
+ removed: number;
+ error: number;
+}
+
+export interface Monitor {
+ id: string;
+ name: string;
+ status: 'active' | 'paused' | 'deleted';
+ schedule: MonitorSchedule;
+ nextRunAt?: string | null;
+ lastRunAt?: string | null;
+ currentCheckId?: string | null;
+ targets: MonitorTarget[];
+ webhook?: MonitorWebhookConfig | null;
+ notification?: MonitorNotification | null;
+ retentionDays: number;
+ estimatedCreditsPerMonth?: number | null;
+ lastCheckSummary?: MonitorSummary | null;
+ createdAt: string;
+ updatedAt: string;
+}
+
+export interface MonitorCheck {
+ id: string;
+ monitorId: string;
+ status:
+ | 'queued'
+ | 'running'
+ | 'completed'
+ | 'failed'
+ | 'partial'
+ | 'skipped_overlap';
+ trigger: 'scheduled' | 'manual';
+ scheduledFor?: string | null;
+ startedAt?: string | null;
+ finishedAt?: string | null;
+ estimatedCredits?: number | null;
+ reservedCredits?: number | null;
+ actualCredits?: number | null;
+ billingStatus:
+ | 'not_applicable'
+ | 'reserved'
+ | 'confirmed'
+ | 'released'
+ | 'failed';
+ summary: MonitorSummary;
+ targetResults?: unknown;
+ notificationStatus?: unknown;
+ error?: string | null;
+ createdAt: string;
+ updatedAt: string;
+}
+
+export interface MonitorCheckPage {
+ id: string;
+ targetId: string;
+ url: string;
+ status: 'same' | 'new' | 'changed' | 'removed' | 'error';
+ previousScrapeId?: string | null;
+ currentScrapeId?: string | null;
+ statusCode?: number | null;
+ error?: string | null;
+ metadata?: unknown;
+ diff?: unknown;
+ createdAt: string;
+}
+
+export interface MonitorCheckDetail extends MonitorCheck {
+ pages: MonitorCheckPage[];
+ next?: string | null;
+}
+
+export interface ListMonitorsOptions {
+ limit?: number;
+ offset?: number;
+}
+
+export type ListMonitorChecksOptions = ListMonitorsOptions;
+
+export type GetMonitorCheckOptions = PaginationConfig & {
+ limit?: number;
+ skip?: number;
+ status?: MonitorCheckPage["status"];
+};
+
export interface ExtractResponse {
success?: boolean;
id?: string;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
index 5dc2cbbdb..842a0ea25 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts
@@ -149,6 +149,20 @@ export class HttpClient {
return this.request<T>({ method: "delete", url: endpoint, headers });
}
+ patch<T = any>(
+ endpoint: string,
+ body: Record<string, unknown>,
+ options?: RequestOptions,
+ ) {
+ return this.request<T>({
+ method: "patch",
+ url: endpoint,
+ data: body,
+ headers: options?.headers,
+ timeout: options?.timeoutMs,
+ });
+ }
+
prepareHeaders(idempotencyKey?: string): Record<string, string> {
const headers: Record<string, string> = {};
if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;
diff --git a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts b/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts
index 4c54d1271..cb8273cfc 100644
--- a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts
+++ b/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts
@@ -2,14 +2,14 @@ import type { HttpClient } from "../utils/httpClient";
import type { Document, PaginationConfig } from "../types";
/**
- * Shared helper to follow `next` cursors and aggregate documents with limits.
+ * Shared helper to follow `next` URLs and aggregate paginated result arrays.
*/
-export async function fetchAllPages(
+export async function fetchAllPages<T = Document>(
http: HttpClient,
nextUrl: string,
- initial: Document[],
+ initial: T[],
pagination?: PaginationConfig
-): Promise<Document[]> {
+): Promise<T[]> {
const docs = initial.slice();
let current: string | null = nextUrl;
let pageCount = 0;
@@ -22,21 +22,24 @@ export async function fetchAllPages(
if (maxPages != null && pageCount >= maxPages) break;
if (maxWaitTime != null && (Date.now() - started) / 1000 > maxWaitTime) break;
- let payload: { success: boolean; next?: string | null; data?: Document[] } | null = null;
+ let payload: { success: boolean; next?: string | null; data?: T[] | { pages?: T[]; next?: string | null } } | null = null;
try {
- const res = await http.get<{ success: boolean; next?: string | null; data?: Document[] }>(current);
+ const res = await http.get<{ success: boolean; next?: string | null; data?: T[] | { pages?: T[]; next?: string | null } }>(current);
payload = res.data;
} catch {
break; // axios rejects on non-2xx; stop pagination gracefully
}
if (!payload?.success) break;
- for (const d of payload.data || []) {
+ const pageData = Array.isArray(payload.data)
+ ? payload.data
+ : payload.data?.pages || [];
+ for (const d of pageData) {
if (maxResults != null && docs.length >= maxResults) break;
- docs.push(d as Document);
+ docs.push(d as T);
}
if (maxResults != null && docs.length >= maxResults) break;
- current = (payload.next ?? null) as string | null;
+ current = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
pageCount += 1;
}
return docs;
diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.php
index e237b51c1..23dbab668 100644
--- a/apps/php-sdk/src/Client/FirecrawlClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlClient.php
@@ -24,6 +24,9 @@ use Firecrawl\Models\CreditUsage;
use Firecrawl\Models\Document;
use Firecrawl\Models\MapData;
use Firecrawl\Models\MapOptions;
+use Firecrawl\Models\Monitor;
+use Firecrawl\Models\MonitorCheck;
+use Firecrawl\Models\MonitorCheckDetail;
use Firecrawl\Models\ParseFile;
use Firecrawl\Models\ParseOptions;
use Firecrawl\Models\ScrapeOptions;
@@ -316,6 +319,142 @@ final class FirecrawlClient
return MapData::fromArray($response['data'] ?? $response);
}
+ // ================================================================
+ // MONITOR
+ // ================================================================
+
+ /**
+ * Create a scheduled monitor.
+ *
+ * @param array<string, mixed> $schedule
+ * @param list<array<string, mixed>> $targets
+ * @param array<string, mixed>|null $webhook
+ * @param array<string, mixed>|null $notification
+ */
+ public function createMonitor(
+ string $name,
+ array $schedule,
+ array $targets,
+ ?array $webhook = null,
+ ?array $notification = null,
+ ?int $retentionDays = null,
+ ): Monitor {
+ $body = array_filter([
+ 'name' => $name,
+ 'schedule' => $schedule,
+ 'targets' => $targets,
+ 'webhook' => $webhook,
+ 'notification' => $notification,
+ 'retentionDays' => $retentionDays,
+ ], static fn ($value) => $value !== null);
+
+ $response = $this->http->post('/v2/monitor', $body);
+
+ return Monitor::fromArray($response['data'] ?? $response);
+ }
+
+ /**
+ * @return list<Monitor>
+ */
+ public function listMonitors(?int $limit = null, ?int $offset = null): array
+ {
+ $response = $this->http->get('/v2/monitor' . $this->query([
+ 'limit' => $limit,
+ 'offset' => $offset,
+ ]));
+
+ return array_map(
+ static fn (array $item): Monitor => Monitor::fromArray($item),
+ $response['data'] ?? [],
+ );
+ }
+
+ public function getMonitor(string $monitorId): Monitor
+ {
+ $response = $this->http->get("/v2/monitor/{$monitorId}");
+
+ return Monitor::fromArray($response['data'] ?? $response);
+ }
+
+ /**
+ * @param array<string, mixed> $attributes
+ */
+ public function updateMonitor(string $monitorId, array $attributes): Monitor
+ {
+ $response = $this->http->patch("/v2/monitor/{$monitorId}", $attributes);
+
+ return Monitor::fromArray($response['data'] ?? $response);
+ }
+
+ public function deleteMonitor(string $monitorId): bool
+ {
+ $response = $this->http->delete("/v2/monitor/{$monitorId}");
+
+ return ($response['success'] ?? false) === true;
+ }
+
+ public function runMonitor(string $monitorId): MonitorCheck
+ {
+ $response = $this->http->post("/v2/monitor/{$monitorId}/run", []);
+
+ return MonitorCheck::fromArray($response['data'] ?? $response);
+ }
+
+ /**
+ * @return list<MonitorCheck>
+ */
+ public function listMonitorChecks(string $monitorId, ?int $limit = null, ?int $offset = null): array
+ {
+ $response = $this->http->get("/v2/monitor/{$monitorId}/checks" . $this->query([
+ 'limit' => $limit,
+ 'offset' => $offset,
+ ]));
+
+ return array_map(
+ static fn (array $item): MonitorCheck => MonitorCheck::fromArray($item),
+ $response['data'] ?? [],
+ );
+ }
+
+ public function getMonitorCheck(
+ string $monitorId,
+ string $checkId,
+ ?int $limit = null,
+ ?int $skip = null,
+ ?string $status = null,
+ bool $autoPaginate = true,
+ ): MonitorCheckDetail {
+ $response = $this->http->get("/v2/monitor/{$monitorId}/checks/{$checkId}" . $this->query([
+ 'limit' => $limit,
+ 'skip' => $skip,
+ 'status' => $status,
+ ]));
+
+ $data = $response['data'] ?? $response;
+ if (isset($response['next'])) {
+ $data['next'] = $response['next'];
+ }
+
+ if (!$autoPaginate) {
+ return MonitorCheckDetail::fromArray($data);
+ }
+
+ while (isset($data['next']) && is_string($data['next']) && $data['next'] !== '') {
+ $this->assertSameOrigin($data['next']);
+ $nextResponse = $this->http->getAbsolute($data['next']);
+ $nextData = $nextResponse['data'] ?? $nextResponse;
+ if (isset($nextResponse['next'])) {
+ $nextData['next'] = $nextResponse['next'];
+ }
+
+ $data['pages'] = array_merge($data['pages'] ?? [], $nextData['pages'] ?? []);
+ $data['next'] = $nextData['next'] ?? null;
+ }
+
+ $data['next'] = null;
+ return MonitorCheckDetail::fromArray($data);
+ }
+
// ================================================================
// SEARCH
// ================================================================
@@ -505,6 +644,16 @@ final class FirecrawlClient
}
}
+ /**
+ * @param array<string, scalar|null> $params
+ */
+ private function query(array $params): string
+ {
+ $params = array_filter($params, static fn ($value) => $value !== null && $value !== '');
+
+ return $params === [] ? '' : '?' . http_build_query($params);
+ }
+
private function pollCrawl(
?string $jobId,
int $pollIntervalSec,
@@ -553,10 +702,29 @@ final class FirecrawlClient
private function assertSameOrigin(string $url): void
{
+ $baseScheme = parse_url($this->http->getBaseUrl(), PHP_URL_SCHEME);
$baseHost = parse_url($this->http->getBaseUrl(), PHP_URL_HOST);
+ $basePort = parse_url($this->http->getBaseUrl(), PHP_URL_PORT);
+ $nextScheme = parse_url($url, PHP_URL_SCHEME);
$nextHost = parse_url($url, PHP_URL_HOST);
-
- if ($baseHost === null || $nextHost === null || strcasecmp($baseHost, $nextHost) !== 0) {
+ $nextPort = parse_url($url, PHP_URL_PORT);
+
+ $basePort ??= is_string($baseScheme) && strcasecmp($baseScheme, 'https') === 0
+ ? 443
+ : 80;
+ $nextPort ??= is_string($nextScheme) && strcasecmp($nextScheme, 'https') === 0
+ ? 443
+ : 80;
+
+ if (
+ $baseScheme === null ||
+ $nextScheme === null ||
+ $baseHost === null ||
+ $nextHost === null ||
+ strcasecmp($baseScheme, $nextScheme) !== 0 ||
+ strcasecmp($baseHost, $nextHost) !== 0 ||
+ $basePort !== $nextPort
+ ) {
throw new FirecrawlException(
'Pagination URL origin does not match the API base URL. Refusing to follow: ' . $url,
);
diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.php
index b878b74e8..fa1d35f5d 100644
--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php
+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php
@@ -67,6 +67,15 @@ final class FirecrawlHttpClient
return $this->request('DELETE', $this->baseUrl . $path);
}
+ /**
+ * @param array<string, mixed> $body
+ * @return array<string, mixed>
+ */
+ public function patch(string $path, array $body): array
+ {
+ return $this->request('PATCH', $this->baseUrl . $path, $body);
+ }
+
/**
* Send a POST request with a multipart/form-data body.
*
@@ -145,7 +154,7 @@ final class FirecrawlHttpClient
if ($multipart !== null) {
$options[RequestOptions::MULTIPART] = $multipart;
- } elseif ($method === 'POST' && $body !== []) {
+ } elseif (($method === 'POST' || $method === 'PATCH') && $body !== []) {
$options[RequestOptions::JSON] = $body;
}
diff --git a/apps/php-sdk/src/Models/Monitor.php b/apps/php-sdk/src/Models/Monitor.php
new file mode 100644
index 000000000..fe4a94d04
--- /dev/null
+++ b/apps/php-sdk/src/Models/Monitor.php
@@ -0,0 +1,76 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Firecrawl\Models;
+
+final class Monitor
+{
+ /**
+ * @param list<array<string, mixed>> $targets
+ * @param array<string, mixed>|null $schedule
+ * @param array<string, mixed>|null $webhook
+ * @param array<string, mixed>|null $notification
+ * @param array<string, mixed>|null $lastCheckSummary
+ */
+ public function __construct(
+ private readonly ?string $id = null,
+ private readonly ?string $name = null,
+ private readonly ?string $status = null,
+ private readonly ?array $schedule = null,
+ private readonly ?string $nextRunAt = null,
+ private readonly ?string $lastRunAt = null,
+ private readonly ?string $currentCheckId = null,
+ private readonly array $targets = [],
+ private readonly ?array $webhook = null,
+ private readonly ?array $notification = null,
+ private readonly ?int $retentionDays = null,
+ private readonly ?int $estimatedCreditsPerMonth = null,
+ private readonly ?array $lastCheckSummary = null,
+ private readonly ?string $createdAt = null,
+ private readonly ?string $updatedAt = null,
+ ) {}
+
+ /** @param array<string, mixed> $data */
+ public static function fromArray(array $data): self
+ {
+ return new self(
+ id: isset($data['id']) ? (string) $data['id'] : null,
+ name: isset($data['name']) ? (string) $data['name'] : null,
+ status: isset($data['status']) ? (string) $data['status'] : null,
+ schedule: isset($data['schedule']) && is_array($data['schedule']) ? $data['schedule'] : null,
+ nextRunAt: isset($data['nextRunAt']) ? (string) $data['nextRunAt'] : null,
+ lastRunAt: isset($data['lastRunAt']) ? (string) $data['lastRunAt'] : null,
+ currentCheckId: isset($data['currentCheckId']) ? (string) $data['currentCheckId'] : null,
+ targets: isset($data['targets']) && is_array($data['targets']) ? $data['targets'] : [],
+ webhook: isset($data['webhook']) && is_array($data['webhook']) ? $data['webhook'] : null,
+ notification: isset($data['notification']) && is_array($data['notification']) ? $data['notification'] : null,
+ retentionDays: isset($data['retentionDays']) ? (int) $data['retentionDays'] : null,
+ estimatedCreditsPerMonth: isset($data['estimatedCreditsPerMonth']) ? (int) $data['estimatedCreditsPerMonth'] : null,
+ lastCheckSummary: isset($data['lastCheckSummary']) && is_array($data['lastCheckSummary']) ? $data['lastCheckSummary'] : null,
+ createdAt: isset($data['createdAt']) ? (string) $data['createdAt'] : null,
+ updatedAt: isset($data['updatedAt']) ? (string) $data['updatedAt'] : null,
+ );
+ }
+
+ public function getId(): ?string { return $this->id; }
+ public function getName(): ?string { return $this->name; }
+ public function getStatus(): ?string { return $this->status; }
+ /** @return array<string, mixed>|null */
+ public function getSchedule(): ?array { return $this->schedule; }
+ public function getNextRunAt(): ?string { return $this->nextRunAt; }
+ public function getLastRunAt(): ?string { return $this->lastRunAt; }
+ public function getCurrentCheckId(): ?string { return $this->currentCheckId; }
+ /** @return list<array<string, mixed>> */
+ public function getTargets(): array { return $this->targets; }
+ /** @return array<string, mixed>|null */
+ public function getWebhook(): ?array { return $this->webhook; }
+ /** @return array<string, mixed>|null */
+ public function getNotification(): ?array { return $this->notification; }
+ public function getRetentionDays(): ?int { return $this->retentionDays; }
+ public function getEstimatedCreditsPerMonth(): ?int { return $this->estimatedCreditsPerMonth; }
+ /** @return array<string, mixed>|null */
+ public function getLastCheckSummary(): ?array { return $this->lastCheckSummary; }
+ public function getCreatedAt(): ?string { return $this->createdAt; }
+ public function getUpdatedAt(): ?string { return $this->updatedAt; }
+}
diff --git a/apps/php-sdk/src/Models/MonitorCheck.php b/apps/php-sdk/src/Models/MonitorCheck.php
new file mode 100644
index 000000000..935f20c04
--- /dev/null
+++ b/apps/php-sdk/src/Models/MonitorCheck.php
@@ -0,0 +1,77 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Firecrawl\Models;
+
+/** @phpstan-consistent-constructor */
+class MonitorCheck
+{
+ /**
+ * @param array<string, mixed> $summary
+ * @param mixed $targetResults
+ * @param mixed $notificationStatus
+ */
+ public function __construct(
+ private readonly ?string $id = null,
+ private readonly ?string $monitorId = null,
+ private readonly ?string $status = null,
+ private readonly ?string $trigger = null,
+ private readonly ?string $scheduledFor = null,
+ private readonly ?string $startedAt = null,
+ private readonly ?string $finishedAt = null,
+ private readonly ?int $estimatedCredits = null,
+ private readonly ?int $reservedCredits = null,
+ private readonly ?int $actualCredits = null,
+ private readonly ?string $billingStatus = null,
+ private readonly array $summary = [],
+ private readonly mixed $targetResults = null,
+ private readonly mixed $notificationStatus = null,
+ private readonly ?string $error = null,
+ private readonly ?string $createdAt = null,
+ private readonly ?string $updatedAt = null,
+ ) {}
+
+ /** @param array<string, mixed> $data */
+ public static function fromArray(array $data): static
+ {
+ return new static(
+ id: isset($data['id']) ? (string) $data['id'] : null,
+ monitorId: isset($data['monitorId']) ? (string) $data['monitorId'] : null,
+ status: isset($data['status']) ? (string) $data['status'] : null,
+ trigger: isset($data['trigger']) ? (string) $data['trigger'] : null,
+ scheduledFor: isset($data['scheduledFor']) ? (string) $data['scheduledFor'] : null,
+ startedAt: isset($data['startedAt']) ? (string) $data['startedAt'] : null,
+ finishedAt: isset($data['finishedAt']) ? (string) $data['finishedAt'] : null,
+ estimatedCredits: isset($data['estimatedCredits']) ? (int) $data['estimatedCredits'] : null,
+ reservedCredits: isset($data['reservedCredits']) ? (int) $data['reservedCredits'] : null,
+ actualCredits: isset($data['actualCredits']) ? (int) $data['actualCredits'] : null,
+ billingStatus: isset($data['billingStatus']) ? (string) $data['billingStatus'] : null,
+ summary: isset($data['summary']) && is_array($data['summary']) ? $data['summary'] : [],
+ targetResults: $data['targetResults'] ?? null,
+ notificationStatus: $data['notificationStatus'] ?? null,
+ error: isset($data['error']) ? (string) $data['error'] : null,
+ createdAt: isset($data['createdAt']) ? (string) $data['createdAt'] : null,
+ updatedAt: isset($data['updatedAt']) ? (string) $data['updatedAt'] : null,
+ );
+ }
+
+ public function getId(): ?string { return $this->id; }
+ public function getMonitorId(): ?string { return $this->monitorId; }
+ public function getStatus(): ?string { return $this->status; }
+ public function getTrigger(): ?string { return $this->trigger; }
+ public function getScheduledFor(): ?string { return $this->scheduledFor; }
+ public function getStartedAt(): ?string { return $this->startedAt; }
+ public function getFinishedAt(): ?string { return $this->finishedAt; }
+ public function getEstimatedCredits(): ?int { return $this->estimatedCredits; }
+ public function getReservedCredits(): ?int { return $this->reservedCredits; }
+ public function getActualCredits(): ?int { return $this->actualCredits; }
+ public function getBillingStatus(): ?string { return $this->billingStatus; }
+ /** @return array<string, mixed> */
+ public function getSummary(): array { return $this->summary; }
+ public function getTargetResults(): mixed { return $this->targetResults; }
+ public function getNotificationStatus(): mixed { return $this->notificationStatus; }
+ public function getError(): ?string { return $this->error; }
+ public function getCreatedAt(): ?string { return $this->createdAt; }
+ public function getUpdatedAt(): ?string { return $this->updatedAt; }
+}
diff --git a/apps/php-sdk/src/Models/MonitorCheckDetail.php b/apps/php-sdk/src/Models/MonitorCheckDetail.php
new file mode 100644
index 000000000..f6709c81e
--- /dev/null
+++ b/apps/php-sdk/src/Models/MonitorCheckDetail.php
@@ -0,0 +1,86 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Firecrawl\Models;
+
+final class MonitorCheckDetail extends MonitorCheck
+{
+ /**
+ * @param list<array<string, mixed>> $pages
+ */
+ public function __construct(
+ ?string $id = null,
+ ?string $monitorId = null,
+ ?string $status = null,
+ ?string $trigger = null,
+ ?string $scheduledFor = null,
+ ?string $startedAt = null,
+ ?string $finishedAt = null,
+ ?int $estimatedCredits = null,
+ ?int $reservedCredits = null,
+ ?int $actualCredits = null,
+ ?string $billingStatus = null,
+ array $summary = [],
+ mixed $targetResults = null,
+ mixed $notificationStatus = null,
+ ?string $error = null,
+ ?string $createdAt = null,
+ ?string $updatedAt = null,
+ private readonly array $pages = [],
+ private readonly ?string $next = null,
+ ) {
+ parent::__construct(
+ id: $id,
+ monitorId: $monitorId,
+ status: $status,
+ trigger: $trigger,
+ scheduledFor: $scheduledFor,
+ startedAt: $startedAt,
+ finishedAt: $finishedAt,
+ estimatedCredits: $estimatedCredits,
+ reservedCredits: $reservedCredits,
+ actualCredits: $actualCredits,
+ billingStatus: $billingStatus,
+ summary: $summary,
+ targetResults: $targetResults,
+ notificationStatus: $notificationStatus,
+ error: $error,
+ createdAt: $createdAt,
+ updatedAt: $updatedAt,
+ );
+ }
+
+ /** @param array<string, mixed> $data */
+ public static function fromArray(array $data): static
+ {
+ /** @var self $check */
+ $check = new self(
+ id: isset($data['id']) ? (string) $data['id'] : null,
+ monitorId: isset($data['monitorId']) ? (string) $data['monitorId'] : null,
+ status: isset($data['status']) ? (string) $data['status'] : null,
+ trigger: isset($data['trigger']) ? (string) $data['trigger'] : null,
+ scheduledFor: isset($data['scheduledFor']) ? (string) $data['scheduledFor'] : null,
+ startedAt: isset($data['startedAt']) ? (string) $data['startedAt'] : null,
+ finishedAt: isset($data['finishedAt']) ? (string) $data['finishedAt'] : null,
+ estimatedCredits: isset($data['estimatedCredits']) ? (int) $data['estimatedCredits'] : null,
+ reservedCredits: isset($data['reservedCredits']) ? (int) $data['reservedCredits'] : null,
+ actualCredits: isset($data['actualCredits']) ? (int) $data['actualCredits'] : null,
+ billingStatus: isset($data['billingStatus']) ? (string) $data['billingStatus'] : null,
+ summary: isset($data['summary']) && is_array($data['summary']) ? $data['summary'] : [],
+ targetResults: $data['targetResults'] ?? null,
+ notificationStatus: $data['notificationStatus'] ?? null,
+ error: isset($data['error']) ? (string) $data['error'] : null,
+ createdAt: isset($data['createdAt']) ? (string) $data['createdAt'] : null,
+ updatedAt: isset($data['updatedAt']) ? (string) $data['updatedAt'] : null,
+ pages: isset($data['pages']) && is_array($data['pages']) ? $data['pages'] : [],
+ next: isset($data['next']) ? (string) $data['next'] : null,
+ );
+
+ return $check;
+ }
+
+ /** @return list<array<string, mixed>> */
+ public function getPages(): array { return $this->pages; }
+ public function getNext(): ?string { return $this->next; }
+}
diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.php
index e7da56ddc..8fb71f9b4 100644
--- a/apps/php-sdk/src/Version.php
+++ b/apps/php-sdk/src/Version.php
@@ -6,5 +6,5 @@ namespace Firecrawl;
final class Version
{
- public const SDK_VERSION = '1.1.1';
+ public const SDK_VERSION = '1.2.0';
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index e42f93db3..8abe1b757 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.24.1"
+__version__ = "4.25.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index 1c21a0a68..00fde14e0 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -92,6 +92,14 @@ class V2Proxy:
self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
self.map = client_instance.map
+ self.create_monitor = client_instance.create_monitor
+ self.list_monitors = client_instance.list_monitors
+ self.get_monitor = client_instance.get_monitor
+ self.update_monitor = client_instance.update_monitor
+ self.delete_monitor = client_instance.delete_monitor
+ self.run_monitor = client_instance.run_monitor
+ self.list_monitor_checks = client_instance.list_monitor_checks
+ self.get_monitor_check = client_instance.get_monitor_check
self.get_concurrency = client_instance.get_concurrency
self.get_credit_usage = client_instance.get_credit_usage
self.get_token_usage = client_instance.get_token_usage
@@ -172,6 +180,14 @@ class AsyncV2Proxy:
self.get_batch_scrape_errors = client_instance.get_batch_scrape_errors
self.map = client_instance.map
+ self.create_monitor = client_instance.create_monitor
+ self.list_monitors = client_instance.list_monitors
+ self.get_monitor = client_instance.get_monitor
+ self.update_monitor = client_instance.update_monitor
+ self.delete_monitor = client_instance.delete_monitor
+ self.run_monitor = client_instance.run_monitor
+ self.list_monitor_checks = client_instance.list_monitor_checks
+ self.get_monitor_check = client_instance.get_monitor_check
self.get_concurrency = client_instance.get_concurrency
self.get_credit_usage = client_instance.get_credit_usage
self.get_token_usage = client_instance.get_token_usage
@@ -242,6 +258,14 @@ class Firecrawl:
self.parse = self._v2_client.parse
self.search = self._v2_client.search
self.map = self._v2_client.map
+ self.create_monitor = self._v2_client.create_monitor
+ self.list_monitors = self._v2_client.list_monitors
+ self.get_monitor = self._v2_client.get_monitor
+ self.update_monitor = self._v2_client.update_monitor
+ self.delete_monitor = self._v2_client.delete_monitor
+ self.run_monitor = self._v2_client.run_monitor
+ self.list_monitor_checks = self._v2_client.list_monitor_checks
+ self.get_monitor_check = self._v2_client.get_monitor_check
self.crawl = self._v2_client.crawl
self.start_crawl = self._v2_client.start_crawl
@@ -336,6 +360,14 @@ class AsyncFirecrawl:
self.parse = self._v2_client.parse
self.search = self._v2_client.search
self.map = self._v2_client.map
+ self.create_monitor = self._v2_client.create_monitor
+ self.list_monitors = self._v2_client.list_monitors
+ self.get_monitor = self._v2_client.get_monitor
+ self.update_monitor = self._v2_client.update_monitor
+ self.delete_monitor = self._v2_client.delete_monitor
+ self.run_monitor = self._v2_client.run_monitor
+ self.list_monitor_checks = self._v2_client.list_monitor_checks
+ self.get_monitor_check = self._v2_client.get_monitor_check
self.start_crawl = self._v2_client.start_crawl
self.get_crawl_status = self._v2_client.get_crawl_status
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 88539b542..17084ccef 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -41,6 +41,14 @@ from .types import (
Location,
PaginationConfig,
AgentOptions,
+ Monitor,
+ MonitorCheck,
+ MonitorCheckDetail,
+ MonitorCreateRequest,
+ MonitorNotification,
+ MonitorSchedule,
+ MonitorTarget,
+ MonitorUpdateRequest,
)
from .utils.http_client import HttpClient
from .utils.error_handler import FirecrawlError
@@ -55,6 +63,7 @@ from .methods import usage as usage_methods
from .methods import extract as extract_module
from .methods import agent as agent_module
from .methods import browser as browser_module
+from .methods import monitor as monitor_module
from .watcher import Watcher
class FirecrawlClient:
@@ -655,6 +664,112 @@ class FirecrawlClient:
) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location]) else None
return map_module.map(self.http_client, url, options)
+
+ def create_monitor(
+ self,
+ name: str,
+ schedule: Union[MonitorSchedule, Dict[str, Any]],
+ targets: List[Union[MonitorTarget, Dict[str, Any]]],
+ *,
+ webhook: Optional[WebhookConfig] = None,
+ notification: Optional[MonitorNotification] = None,
+ retention_days: Optional[int] = None,
+ ) -> Monitor:
+ """Create a scheduled monitor."""
+ if isinstance(schedule, dict):
+ schedule = MonitorSchedule(**schedule)
+ request = MonitorCreateRequest(
+ name=name,
+ schedule=schedule,
+ targets=targets,
+ webhook=webhook,
+ notification=notification,
+ retention_days=retention_days,
+ )
+ return monitor_module.create_monitor(self.http_client, request)
+
+ def list_monitors(
+ self,
+ *,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ ) -> List[Monitor]:
+ """List monitors for the authenticated team."""
+ return monitor_module.list_monitors(self.http_client, limit=limit, offset=offset)
+
+ def get_monitor(self, monitor_id: str) -> Monitor:
+ """Get a monitor by ID."""
+ return monitor_module.get_monitor(self.http_client, monitor_id)
+
+ def update_monitor(
+ self,
+ monitor_id: str,
+ *,
+ name: Optional[str] = None,
+ status: Optional[Literal["active", "paused"]] = None,
+ schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,
+ webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,
+ notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,
+ targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,
+ retention_days: Optional[int] = None,
+ ) -> Monitor:
+ """Update a monitor."""
+ if isinstance(schedule, dict):
+ schedule = MonitorSchedule(**schedule)
+ request = MonitorUpdateRequest(
+ name=name,
+ status=status,
+ schedule=schedule,
+ webhook=webhook,
+ notification=notification,
+ targets=targets,
+ retention_days=retention_days,
+ )
+ return monitor_module.update_monitor(self.http_client, monitor_id, request)
+
+ def delete_monitor(self, monitor_id: str) -> bool:
+ """Delete a monitor."""
+ return monitor_module.delete_monitor(self.http_client, monitor_id)
+
+ def run_monitor(self, monitor_id: str) -> MonitorCheck:
+ """Run a monitor manually."""
+ return monitor_module.run_monitor(self.http_client, monitor_id)
+
+ def list_monitor_checks(
+ self,
+ monitor_id: str,
+ *,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ ) -> List[MonitorCheck]:
+ """List checks for a monitor."""
+ return monitor_module.list_monitor_checks(
+ self.http_client,
+ monitor_id,
+ limit=limit,
+ offset=offset,
+ )
+
+ def get_monitor_check(
+ self,
+ monitor_id: str,
+ check_id: str,
+ *,
+ limit: Optional[int] = None,
+ skip: Optional[int] = None,
+ status: Optional[Literal["same", "new", "changed", "removed", "error"]] = None,
+ pagination_config: Optional[PaginationConfig] = None,
+ ) -> MonitorCheckDetail:
+ """Get a monitor check with page results, auto-paginated by default."""
+ return monitor_module.get_monitor_check(
+ self.http_client,
+ monitor_id,
+ check_id,
+ limit=limit,
+ skip=skip,
+ status=status,
+ pagination_config=pagination_config,
+ )
def cancel_crawl(self, crawl_id: str) -> bool:
"""
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index b41da8eff..834ac4407 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -36,6 +36,14 @@ from .types import (
PDFAction,
Location,
PaginationConfig,
+ Monitor,
+ MonitorCheck,
+ MonitorCheckDetail,
+ MonitorCreateRequest,
+ MonitorNotification,
+ MonitorSchedule,
+ MonitorTarget,
+ MonitorUpdateRequest,
)
from .utils.http_client import HttpClient
from .utils.http_client_async import AsyncHttpClient
@@ -50,6 +58,7 @@ from .methods.aio import usage as async_usage # type: ignore[attr-defined]
from .methods.aio import extract as async_extract # type: ignore[attr-defined]
from .methods.aio import agent as async_agent # type: ignore[attr-defined]
from .methods.aio import browser as async_browser # type: ignore[attr-defined]
+from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]
from .watcher_async import AsyncWatcher
@@ -332,6 +341,112 @@ class AsyncFirecrawlClient:
) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout]) else None
return await async_map.map(self.async_http_client, url, options)
+ async def create_monitor(
+ self,
+ name: str,
+ schedule: Union[MonitorSchedule, Dict[str, Any]],
+ targets: List[Union[MonitorTarget, Dict[str, Any]]],
+ *,
+ webhook: Optional[WebhookConfig] = None,
+ notification: Optional[MonitorNotification] = None,
+ retention_days: Optional[int] = None,
+ ) -> Monitor:
+ if isinstance(schedule, dict):
+ schedule = MonitorSchedule(**schedule)
+ request = MonitorCreateRequest(
+ name=name,
+ schedule=schedule,
+ targets=targets,
+ webhook=webhook,
+ notification=notification,
+ retention_days=retention_days,
+ )
+ return await async_monitor.create_monitor(self.async_http_client, request)
+
+ async def list_monitors(
+ self,
+ *,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ ) -> List[Monitor]:
+ return await async_monitor.list_monitors(
+ self.async_http_client,
+ limit=limit,
+ offset=offset,
+ )
+
+ async def get_monitor(self, monitor_id: str) -> Monitor:
+ return await async_monitor.get_monitor(self.async_http_client, monitor_id)
+
+ async def update_monitor(
+ self,
+ monitor_id: str,
+ *,
+ name: Optional[str] = None,
+ status: Optional[Literal["active", "paused"]] = None,
+ schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,
+ webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,
+ notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,
+ targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,
+ retention_days: Optional[int] = None,
+ ) -> Monitor:
+ if isinstance(schedule, dict):
+ schedule = MonitorSchedule(**schedule)
+ request = MonitorUpdateRequest(
+ name=name,
+ status=status,
+ schedule=schedule,
+ webhook=webhook,
+ notification=notification,
+ targets=targets,
+ retention_days=retention_days,
+ )
+ return await async_monitor.update_monitor(
+ self.async_http_client,
+ monitor_id,
+ request,
+ )
+
+ async def delete_monitor(self, monitor_id: str) -> bool:
+ return await async_monitor.delete_monitor(self.async_http_client, monitor_id)
+
+ async def run_monitor(self, monitor_id: str) -> MonitorCheck:
+ return await async_monitor.run_monitor(self.async_http_client, monitor_id)
+
+ async def list_monitor_checks(
+ self,
+ monitor_id: str,
+ *,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+ ) -> List[MonitorCheck]:
+ return await async_monitor.list_monitor_checks(
+ self.async_http_client,
+ monitor_id,
+ limit=limit,
+ offset=offset,
+ )
+
+ async def get_monitor_check(
+ self,
+ monitor_id: str,
+ check_id: str,
+ *,
+ limit: Optional[int] = None,
+ skip: Optional[int] = None,
+ status: Optional[Literal["same", "new", "changed", "removed", "error"]] = None,
+ pagination_config: Optional[PaginationConfig] = None,
+ ) -> MonitorCheckDetail:
+ return await async_monitor.get_monitor_check(
+ self.async_http_client,
+ monitor_id,
+ check_id,
+ limit=limit,
+ skip=skip,
+ status=status,
+ pagination_config=pagination_config,
+ )
+
async def start_batch_scrape(self, urls: List[str], **kwargs) -> Any:
return await async_batch.start_batch_scrape(self.async_http_client, urls, **kwargs)
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/monitor.py b/apps/python-sdk/firecrawl/v2/methods/aio/monitor.py
new file mode 100644
index 000000000..fe98ffa05
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/monitor.py
@@ -0,0 +1,215 @@
+import time
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel
+
+from ...types import (
+ Monitor,
+ MonitorCheck,
+ MonitorCheckDetail,
+ MonitorCheckPage,
+ MonitorCreateRequest,
+ PaginationConfig,
+ MonitorTarget,
+ MonitorUpdateRequest,
+ ScrapeOptions,
+)
+from ...utils.error_handler import handle_response_error
+from ...utils.http_client_async import AsyncHttpClient
+from ...utils.validation import prepare_scrape_options
+
+
+def _dump(value: Any) -> Any:
+ if isinstance(value, ScrapeOptions):
+ return prepare_scrape_options(value)
+ if isinstance(value, MonitorTarget):
+ data = value.model_dump(exclude_none=True, by_alias=True)
+ if isinstance(value.scrape_options, ScrapeOptions):
+ data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)
+ return _prepare_target(data)
+ if isinstance(value, BaseModel):
+ return value.model_dump(exclude_none=True, by_alias=True)
+ if isinstance(value, list):
+ return [_dump(item) for item in value]
+ if isinstance(value, dict):
+ return {key: _dump(item) for key, item in value.items() if item is not None}
+ return value
+
+
+def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:
+ prepared = dict(target)
+ if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):
+ prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])
+ if "crawlOptions" in prepared:
+ prepared["crawlOptions"] = _dump(prepared["crawlOptions"])
+ return prepared
+
+
+def _prepare_payload(request: Any) -> Dict[str, Any]:
+ payload = _dump(request)
+ if not isinstance(payload, dict):
+ raise ValueError("Monitor request must be an object")
+ if "targets" in payload:
+ payload["targets"] = [
+ _prepare_target(_dump(target))
+ for target in payload.get("targets", [])
+ ]
+ return payload
+
+
+async def _data_or_error(response, action: str) -> Any:
+ if response.status_code >= 400:
+ handle_response_error(response, action)
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+ return body.get("data")
+
+
+async def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:
+ if response.status_code >= 400:
+ handle_response_error(response, action)
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+ data = body.get("data") or {}
+ if body.get("next") is not None:
+ data["next"] = body.get("next")
+ return data
+
+
+async def _fetch_all_monitor_check_pages(
+ client: AsyncHttpClient,
+ next_url: str,
+ initial_pages: List[MonitorCheckPage],
+ pagination_config: Optional[PaginationConfig] = None,
+) -> List[MonitorCheckPage]:
+ pages = initial_pages.copy()
+ current_url = next_url
+ page_count = 0
+ max_pages = pagination_config.max_pages if pagination_config else None
+ max_results = pagination_config.max_results if pagination_config else None
+ max_wait_time = pagination_config.max_wait_time if pagination_config else None
+ start_time = time.monotonic()
+
+ while current_url:
+ if max_pages is not None and page_count >= max_pages:
+ break
+ if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:
+ break
+
+ response = await client.get(current_url)
+ if response.status_code >= 400:
+ break
+ try:
+ data = await _monitor_check_data_or_error(response, "get monitor check page")
+ except Exception:
+ break
+
+ for page in data.get("pages") or []:
+ if max_results is not None and len(pages) >= max_results:
+ break
+ pages.append(MonitorCheckPage(**page))
+
+ if max_results is not None and len(pages) >= max_results:
+ break
+
+ current_url = data.get("next")
+ page_count += 1
+
+ return pages
+
+
+async def create_monitor(client: AsyncHttpClient, request: MonitorCreateRequest) -> Monitor:
+ data = await _data_or_error(await client.post("/v2/monitor", _prepare_payload(request)), "create monitor")
+ return Monitor(**data)
+
+
+async def list_monitors(client: AsyncHttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:
+ params = []
+ if limit is not None:
+ params.append(f"limit={limit}")
+ if offset is not None:
+ params.append(f"offset={offset}")
+ suffix = f"?{'&'.join(params)}" if params else ""
+ data = await _data_or_error(await client.get(f"/v2/monitor{suffix}"), "list monitors")
+ return [Monitor(**item) for item in data or []]
+
+
+async def get_monitor(client: AsyncHttpClient, monitor_id: str) -> Monitor:
+ data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}"), "get monitor")
+ return Monitor(**data)
+
+
+async def update_monitor(client: AsyncHttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:
+ data = await _data_or_error(await client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")
+ return Monitor(**data)
+
+
+async def delete_monitor(client: AsyncHttpClient, monitor_id: str) -> bool:
+ response = await client.delete(f"/v2/monitor/{monitor_id}")
+ if response.status_code >= 400:
+ handle_response_error(response, "delete monitor")
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+ return True
+
+
+async def run_monitor(client: AsyncHttpClient, monitor_id: str) -> MonitorCheck:
+ data = await _data_or_error(await client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")
+ return MonitorCheck(**data)
+
+
+async def list_monitor_checks(
+ client: AsyncHttpClient,
+ monitor_id: str,
+ *,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+) -> List[MonitorCheck]:
+ params = []
+ if limit is not None:
+ params.append(f"limit={limit}")
+ if offset is not None:
+ params.append(f"offset={offset}")
+ suffix = f"?{'&'.join(params)}" if params else ""
+ data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")
+ return [MonitorCheck(**item) for item in data or []]
+
+
+async def get_monitor_check(
+ client: AsyncHttpClient,
+ monitor_id: str,
+ check_id: str,
+ *,
+ limit: Optional[int] = None,
+ skip: Optional[int] = None,
+ status: Optional[str] = None,
+ pagination_config: Optional[PaginationConfig] = None,
+) -> MonitorCheckDetail:
+ params = []
+ if limit is not None:
+ params.append(f"limit={limit}")
+ if skip is not None:
+ params.append(f"skip={skip}")
+ if status is not None:
+ params.append(f"status={status}")
+ suffix = f"?{'&'.join(params)}" if params else ""
+ data = await _monitor_check_data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")
+ detail = MonitorCheckDetail(**data)
+
+ auto_paginate = pagination_config.auto_paginate if pagination_config else True
+ if auto_paginate and detail.next and not (
+ pagination_config
+ and pagination_config.max_results is not None
+ and len(detail.pages) >= pagination_config.max_results
+ ):
+ detail.pages = await _fetch_all_monitor_check_pages(
+ client,
+ detail.next,
+ detail.pages,
+ pagination_config,
+ )
+ detail.next = None
+
+ return detail
diff --git a/apps/python-sdk/firecrawl/v2/methods/monitor.py b/apps/python-sdk/firecrawl/v2/methods/monitor.py
new file mode 100644
index 000000000..2164206b0
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/monitor.py
@@ -0,0 +1,214 @@
+import time
+from typing import Any, Dict, List, Optional
+from pydantic import BaseModel
+
+from ..types import (
+ Monitor,
+ MonitorCheck,
+ MonitorCheckDetail,
+ MonitorCheckPage,
+ MonitorCreateRequest,
+ PaginationConfig,
+ MonitorTarget,
+ MonitorUpdateRequest,
+ ScrapeOptions,
+)
+from ..utils import HttpClient, handle_response_error
+from ..utils.validation import prepare_scrape_options
+
+
+def _dump(value: Any) -> Any:
+ if isinstance(value, ScrapeOptions):
+ return prepare_scrape_options(value)
+ if isinstance(value, MonitorTarget):
+ data = value.model_dump(exclude_none=True, by_alias=True)
+ if isinstance(value.scrape_options, ScrapeOptions):
+ data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)
+ return _prepare_target(data)
+ if isinstance(value, BaseModel):
+ return value.model_dump(exclude_none=True, by_alias=True)
+ if isinstance(value, list):
+ return [_dump(item) for item in value]
+ if isinstance(value, dict):
+ return {key: _dump(item) for key, item in value.items() if item is not None}
+ return value
+
+
+def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:
+ prepared = dict(target)
+ if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):
+ prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])
+ if "crawlOptions" in prepared:
+ prepared["crawlOptions"] = _dump(prepared["crawlOptions"])
+ return prepared
+
+
+def _prepare_payload(request: Any) -> Dict[str, Any]:
+ payload = _dump(request)
+ if not isinstance(payload, dict):
+ raise ValueError("Monitor request must be an object")
+ if "targets" in payload:
+ payload["targets"] = [
+ _prepare_target(_dump(target))
+ for target in payload.get("targets", [])
+ ]
+ return payload
+
+
+def _data_or_error(response, action: str) -> Any:
+ if not response.ok:
+ handle_response_error(response, action)
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+ return body.get("data")
+
+
+def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:
+ if not response.ok:
+ handle_response_error(response, action)
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+ data = body.get("data") or {}
+ if body.get("next") is not None:
+ data["next"] = body.get("next")
+ return data
+
+
+def _fetch_all_monitor_check_pages(
+ client: HttpClient,
+ next_url: str,
+ initial_pages: List[MonitorCheckPage],
+ pagination_config: Optional[PaginationConfig] = None,
+) -> List[MonitorCheckPage]:
+ pages = initial_pages.copy()
+ current_url = next_url
+ page_count = 0
+ max_pages = pagination_config.max_pages if pagination_config else None
+ max_results = pagination_config.max_results if pagination_config else None
+ max_wait_time = pagination_config.max_wait_time if pagination_config else None
+ start_time = time.monotonic()
+
+ while current_url:
+ if max_pages is not None and page_count >= max_pages:
+ break
+ if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:
+ break
+
+ response = client.get(current_url)
+ if not response.ok:
+ break
+ try:
+ data = _monitor_check_data_or_error(response, "get monitor check page")
+ except Exception:
+ break
+
+ for page in data.get("pages") or []:
+ if max_results is not None and len(pages) >= max_results:
+ break
+ pages.append(MonitorCheckPage(**page))
+
+ if max_results is not None and len(pages) >= max_results:
+ break
+
+ current_url = data.get("next")
+ page_count += 1
+
+ return pages
+
+
+def create_monitor(client: HttpClient, request: MonitorCreateRequest) -> Monitor:
+ data = _data_or_error(client.post("/v2/monitor", _prepare_payload(request)), "create monitor")
+ return Monitor(**data)
+
+
+def list_monitors(client: HttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:
+ params = []
+ if limit is not None:
+ params.append(f"limit={limit}")
+ if offset is not None:
+ params.append(f"offset={offset}")
+ suffix = f"?{'&'.join(params)}" if params else ""
+ data = _data_or_error(client.get(f"/v2/monitor{suffix}"), "list monitors")
+ return [Monitor(**item) for item in data or []]
+
+
+def get_monitor(client: HttpClient, monitor_id: str) -> Monitor:
+ data = _data_or_error(client.get(f"/v2/monitor/{monitor_id}"), "get monitor")
+ return Monitor(**data)
+
+
+def update_monitor(client: HttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:
+ data = _data_or_error(client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")
+ return Monitor(**data)
+
+
+def delete_monitor(client: HttpClient, monitor_id: str) -> bool:
+ response = client.delete(f"/v2/monitor/{monitor_id}")
+ if not response.ok:
+ handle_response_error(response, "delete monitor")
+ body = response.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error occurred"))
+ return True
+
+
+def run_monitor(client: HttpClient, monitor_id: str) -> MonitorCheck:
+ data = _data_or_error(client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")
+ return MonitorCheck(**data)
+
+
+def list_monitor_checks(
+ client: HttpClient,
+ monitor_id: str,
+ *,
+ limit: Optional[int] = None,
+ offset: Optional[int] = None,
+) -> List[MonitorCheck]:
+ params = []
+ if limit is not None:
+ params.append(f"limit={limit}")
+ if offset is not None:
+ params.append(f"offset={offset}")
+ suffix = f"?{'&'.join(params)}" if params else ""
+ data = _data_or_error(client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")
+ return [MonitorCheck(**item) for item in data or []]
+
+
+def get_monitor_check(
+ client: HttpClient,
+ monitor_id: str,
+ check_id: str,
+ *,
+ limit: Optional[int] = None,
+ skip: Optional[int] = None,
+ status: Optional[str] = None,
+ pagination_config: Optional[PaginationConfig] = None,
+) -> MonitorCheckDetail:
+ params = []
+ if limit is not None:
+ params.append(f"limit={limit}")
+ if skip is not None:
+ params.append(f"skip={skip}")
+ if status is not None:
+ params.append(f"status={status}")
+ suffix = f"?{'&'.join(params)}" if params else ""
+ data = _monitor_check_data_or_error(client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")
+ detail = MonitorCheckDetail(**data)
+
+ auto_paginate = pagination_config.auto_paginate if pagination_config else True
+ if auto_paginate and detail.next and not (
+ pagination_config
+ and pagination_config.max_results is not None
+ and len(detail.pages) >= pagination_config.max_results
+ ):
+ detail.pages = _fetch_all_monitor_check_pages(
+ client,
+ detail.next,
+ detail.pages,
+ pagination_config,
+ )
+ detail.next = None
+
+ return detail
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 9b06f0341..a2fe2368f 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -797,6 +797,136 @@ class MapResponse(BaseResponse[MapData]):
pass
+# Monitor types
+class MonitorSchedule(BaseModel):
+ """Cron schedule for a monitor."""
+
+ cron: str
+ timezone: str = "UTC"
+
+
+class MonitorEmailNotification(BaseModel):
+ enabled: bool = False
+ recipients: List[str] = []
+ include_diffs: bool = Field(default=False, alias="includeDiffs")
+
+ model_config = {"populate_by_name": True}
+
+
+class MonitorNotification(BaseModel):
+ email: Optional[MonitorEmailNotification] = None
+
+
+class MonitorTarget(BaseModel):
+ """A scrape or crawl target stored on a monitor."""
+
+ model_config = {"extra": "allow", "populate_by_name": True}
+
+ id: Optional[str] = None
+ type: Literal["scrape", "crawl"]
+ urls: Optional[List[str]] = None
+ url: Optional[str] = None
+ scrape_options: Optional[Union[ScrapeOptions, Dict[str, Any]]] = Field(default=None, alias="scrapeOptions")
+ crawl_options: Optional[Dict[str, Any]] = Field(default=None, alias="crawlOptions")
+
+
+class MonitorCreateRequest(BaseModel):
+ model_config = {"populate_by_name": True}
+
+ name: str
+ schedule: MonitorSchedule
+ webhook: Optional[WebhookConfig] = None
+ notification: Optional[MonitorNotification] = None
+ targets: List[Union[MonitorTarget, Dict[str, Any]]]
+ retention_days: Optional[int] = Field(default=None, alias="retentionDays")
+
+
+class MonitorUpdateRequest(BaseModel):
+ model_config = {"populate_by_name": True}
+
+ name: Optional[str] = None
+ status: Optional[Literal["active", "paused"]] = None
+ schedule: Optional[MonitorSchedule] = None
+ webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None
+ notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None
+ targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None
+ retention_days: Optional[int] = Field(default=None, alias="retentionDays")
+
+
+class MonitorSummary(BaseModel):
+ total_pages: int = Field(default=0, alias="totalPages")
+ same: int = 0
+ changed: int = 0
+ new: int = 0
+ removed: int = 0
+ error: int = 0
+
+ model_config = {"populate_by_name": True}
+
+
+class Monitor(BaseModel):
+ model_config = {"populate_by_name": True, "extra": "allow"}
+
+ id: str
+ name: str
+ status: Literal["active", "paused", "deleted"]
+ schedule: MonitorSchedule
+ next_run_at: Optional[str] = Field(default=None, alias="nextRunAt")
+ last_run_at: Optional[str] = Field(default=None, alias="lastRunAt")
+ current_check_id: Optional[str] = Field(default=None, alias="currentCheckId")
+ targets: List[Dict[str, Any]]
+ webhook: Optional[Dict[str, Any]] = None
+ notification: Optional[Dict[str, Any]] = None
+ retention_days: int = Field(alias="retentionDays")
+ estimated_credits_per_month: Optional[int] = Field(default=None, alias="estimatedCreditsPerMonth")
+ last_check_summary: Optional[MonitorSummary] = Field(default=None, alias="lastCheckSummary")
+ created_at: str = Field(alias="createdAt")
+ updated_at: str = Field(alias="updatedAt")
+
+
+class MonitorCheck(BaseModel):
+ model_config = {"populate_by_name": True, "extra": "allow"}
+
+ id: str
+ monitor_id: str = Field(alias="monitorId")
+ status: Literal["queued", "running", "completed", "failed", "partial", "skipped_overlap"]
+ trigger: Literal["scheduled", "manual"]
+ scheduled_for: Optional[str] = Field(default=None, alias="scheduledFor")
+ started_at: Optional[str] = Field(default=None, alias="startedAt")
+ finished_at: Optional[str] = Field(default=None, alias="finishedAt")
+ estimated_credits: Optional[int] = Field(default=None, alias="estimatedCredits")
+ reserved_credits: Optional[int] = Field(default=None, alias="reservedCredits")
+ actual_credits: Optional[int] = Field(default=None, alias="actualCredits")
+ billing_status: Literal["not_applicable", "reserved", "confirmed", "released", "failed"] = Field(alias="billingStatus")
+ summary: MonitorSummary
+ target_results: Optional[Any] = Field(default=None, alias="targetResults")
+ notification_status: Optional[Any] = Field(default=None, alias="notificationStatus")
+ error: Optional[str] = None
+ created_at: str = Field(alias="createdAt")
+ updated_at: str = Field(alias="updatedAt")
+
+
+class MonitorCheckPage(BaseModel):
+ model_config = {"populate_by_name": True, "extra": "allow"}
+
+ id: str
+ target_id: str = Field(alias="targetId")
+ url: str
+ status: Literal["same", "new", "changed", "removed", "error"]
+ previous_scrape_id: Optional[str] = Field(default=None, alias="previousScrapeId")
+ current_scrape_id: Optional[str] = Field(default=None, alias="currentScrapeId")
+ status_code: Optional[int] = Field(default=None, alias="statusCode")
+ error: Optional[str] = None
+ metadata: Optional[Any] = None
+ diff: Optional[Any] = None
+ created_at: str = Field(alias="createdAt")
+
+
+class MonitorCheckDetail(MonitorCheck):
+ pages: List[MonitorCheckPage] = []
+ next: Optional[str] = None
+
+
# Extract types
class ExtractRequest(BaseModel):
"""Request for extract operations."""
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.py
index cc4168246..7a75ddbb3 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py
@@ -263,4 +263,50 @@ class HttpClient:
time.sleep(backoff_factor * (2 ** attempt))
# This should never be reached due to the exception handling above
- raise last_exception or Exception("Unexpected error in DELETE request")
\ No newline at end of file
+ raise last_exception or Exception("Unexpected error in DELETE request")
+
+ def patch(
+ self,
+ endpoint: str,
+ data: Dict[str, Any],
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
+ ) -> requests.Response:
+ """Make a PATCH request with retry logic."""
+ if headers is None:
+ headers = self._prepare_headers()
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
+
+ payload = dict(data)
+ payload['origin'] = f'python-sdk@{version}'
+ url = self._build_url(endpoint)
+
+ last_exception = None
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
+ try:
+ response = requests.patch(
+ url,
+ json=payload,
+ headers=headers,
+ timeout=timeout
+ )
+ if response.status_code == 502 and attempt < num_attempts - 1:
+ time.sleep(backoff_factor * (2 ** attempt))
+ continue
+ return response
+ except requests.RequestException as e:
+ last_exception = e
+ if attempt == num_attempts - 1:
+ raise e
+ time.sleep(backoff_factor * (2 ** attempt))
+
+ raise last_exception or Exception("Unexpected error in PATCH request")
\ No newline at end of file
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
index 9c8e7dec1..52d14623f 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
@@ -201,3 +201,45 @@ class AsyncHttpClient:
await asyncio.sleep(backoff_factor * (2 ** attempt))
raise last_exception or Exception("Unexpected error in DELETE request")
+
+ async def patch(
+ self,
+ endpoint: str,
+ data: Dict[str, Any],
+ headers: Optional[Dict[str, str]] = None,
+ timeout: Optional[float] = None,
+ retries: Optional[int] = None,
+ backoff_factor: Optional[float] = None,
+ ) -> httpx.Response:
+ if timeout is None:
+ timeout = self.timeout
+ if retries is None:
+ retries = self.max_retries
+ if backoff_factor is None:
+ backoff_factor = self.backoff_factor
+
+ payload = dict(data)
+ payload["origin"] = f"python-sdk@{version}"
+
+ last_exception = None
+ num_attempts = max(1, retries)
+
+ for attempt in range(num_attempts):
+ try:
+ response = await self._client.patch(
+ endpoint,
+ json=payload,
+ headers={**self._headers(), **(headers or {})},
+ timeout=timeout,
+ )
+ if response.status_code == 502 and attempt < num_attempts - 1:
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+ continue
+ return response
+ except httpx.HTTPError as e:
+ last_exception = e
+ if attempt == num_attempts - 1:
+ raise e
+ await asyncio.sleep(backoff_factor * (2 ** attempt))
+
+ raise last_exception or Exception("Unexpected error in PATCH request")
diff --git a/apps/ruby-sdk/lib/firecrawl.rb b/apps/ruby-sdk/lib/firecrawl.rb
index 79af43268..64cd6dd5a 100644
--- a/apps/ruby-sdk/lib/firecrawl.rb
+++ b/apps/ruby-sdk/lib/firecrawl.rb
@@ -23,4 +23,5 @@ require_relative "firecrawl/models/agent_response"
require_relative "firecrawl/models/agent_status_response"
require_relative "firecrawl/models/concurrency_check"
require_relative "firecrawl/models/credit_usage"
+require_relative "firecrawl/models/monitor"
require_relative "firecrawl/client"
diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rb
index 635c1bbba..7eaff8325 100644
--- a/apps/ruby-sdk/lib/firecrawl/client.rb
+++ b/apps/ruby-sdk/lib/firecrawl/client.rb
@@ -1,6 +1,7 @@
# frozen_string_literal: true
require "json"
+require "uri"
module Firecrawl
# Client for the Firecrawl v2 API.
@@ -280,6 +281,83 @@ module Firecrawl
Models::MapData.new(data)
end
+ # ================================================================
+ # MONITOR
+ # ================================================================
+
+ def create_monitor(name:, schedule:, targets:, webhook: nil, notification: nil, retention_days: nil)
+ body = {
+ "name" => name,
+ "schedule" => schedule,
+ "targets" => targets,
+ "webhook" => webhook,
+ "notification" => notification,
+ "retentionDays" => retention_days,
+ }.compact
+ raw = @http.post("/v2/monitor", body)
+ Models::Monitor.new(raw["data"] || raw)
+ end
+
+ def list_monitors(limit: nil, offset: nil)
+ raw = @http.get("/v2/monitor#{query(limit: limit, offset: offset)}")
+ (raw["data"] || []).map { |item| Models::Monitor.new(item) }
+ end
+
+ def get_monitor(monitor_id)
+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?
+
+ raw = @http.get("/v2/monitor/#{monitor_id}")
+ Models::Monitor.new(raw["data"] || raw)
+ end
+
+ def update_monitor(monitor_id, **attrs)
+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?
+
+ body = {
+ "name" => attrs[:name],
+ "status" => attrs[:status],
+ "schedule" => attrs[:schedule],
+ "webhook" => attrs[:webhook],
+ "notification" => attrs[:notification],
+ "targets" => attrs[:targets],
+ "retentionDays" => attrs[:retention_days],
+ }.compact
+ raw = @http.patch("/v2/monitor/#{monitor_id}", body)
+ Models::Monitor.new(raw["data"] || raw)
+ end
+
+ def delete_monitor(monitor_id)
+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?
+
+ @http.delete("/v2/monitor/#{monitor_id}")["success"] == true
+ end
+
+ def run_monitor(monitor_id)
+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?
+
+ raw = @http.post("/v2/monitor/#{monitor_id}/run", {})
+ Models::MonitorCheck.new(raw["data"] || raw)
+ end
+
+ def list_monitor_checks(monitor_id, limit: nil, offset: nil)
+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?
+
+ raw = @http.get("/v2/monitor/#{monitor_id}/checks#{query(limit: limit, offset: offset)}")
+ (raw["data"] || []).map { |item| Models::MonitorCheck.new(item) }
+ end
+
+ def get_monitor_check(monitor_id, check_id, limit: nil, skip: nil, status: nil, auto_paginate: true)
+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?
+ raise ArgumentError, "Check ID is required" if check_id.nil?
+
+ params = query(limit: limit, skip: skip, status: status)
+ raw = @http.get("/v2/monitor/#{monitor_id}/checks/#{check_id}#{params}")
+ data = raw["data"] || raw
+ data["next"] = raw["next"] if raw["next"]
+ check = Models::MonitorCheckDetail.new(data)
+ auto_paginate ? paginate_monitor_check(check) : check
+ end
+
# ================================================================
# SEARCH
# ================================================================
@@ -378,6 +456,11 @@ module Firecrawl
private
+ def query(**params)
+ compact = params.compact
+ compact.empty? ? "" : "?#{URI.encode_www_form(compact)}"
+ end
+
def poll_crawl(job_id, poll_interval, timeout)
deadline = Time.now + timeout
while Time.now < deadline
@@ -423,5 +506,20 @@ module Firecrawl
end
job
end
+
+ def paginate_monitor_check(check)
+ check.pages ||= []
+ current = check
+ while current.next_url && !current.next_url.empty?
+ raw = @http.get_absolute(current.next_url)
+ data = raw["data"] || raw
+ data["next"] = raw["next"] if raw["next"]
+ next_page = Models::MonitorCheckDetail.new(data)
+ check.pages.concat(next_page.pages) unless next_page.pages.empty?
+ current = next_page
+ end
+ check.next_url = nil
+ check
+ end
end
end
diff --git a/apps/ruby-sdk/lib/firecrawl/http_client.rb b/apps/ruby-sdk/lib/firecrawl/http_client.rb
index 248619e5d..4d1a819e0 100644
--- a/apps/ruby-sdk/lib/firecrawl/http_client.rb
+++ b/apps/ruby-sdk/lib/firecrawl/http_client.rb
@@ -59,6 +59,16 @@ module Firecrawl
execute_with_retry(uri, request)
end
+ # Sends a PATCH request with JSON body.
+ def patch(path, body)
+ uri = URI("#{@base_url}#{path}")
+ request = Net::HTTP::Patch.new(uri)
+ request["Authorization"] = "Bearer #{@api_key}"
+ request["Content-Type"] = "application/json"
+ request.body = JSON.generate(body)
+ execute_with_retry(uri, request)
+ end
+
# Sends a POST request with a multipart/form-data body.
#
# @param path [String] API path
diff --git a/apps/ruby-sdk/lib/firecrawl/models/monitor.rb b/apps/ruby-sdk/lib/firecrawl/models/monitor.rb
new file mode 100644
index 000000000..8948baf23
--- /dev/null
+++ b/apps/ruby-sdk/lib/firecrawl/models/monitor.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Firecrawl
+ module Models
+ class Monitor
+ attr_reader :id, :name, :status, :schedule, :next_run_at, :last_run_at,
+ :current_check_id, :targets, :webhook, :notification,
+ :retention_days, :estimated_credits_per_month,
+ :last_check_summary, :created_at, :updated_at
+
+ def initialize(data)
+ @id = data["id"]
+ @name = data["name"]
+ @status = data["status"]
+ @schedule = data["schedule"]
+ @next_run_at = data["nextRunAt"]
+ @last_run_at = data["lastRunAt"]
+ @current_check_id = data["currentCheckId"]
+ @targets = data["targets"] || []
+ @webhook = data["webhook"]
+ @notification = data["notification"]
+ @retention_days = data["retentionDays"]
+ @estimated_credits_per_month = data["estimatedCreditsPerMonth"]
+ @last_check_summary = data["lastCheckSummary"]
+ @created_at = data["createdAt"]
+ @updated_at = data["updatedAt"]
+ end
+ end
+
+ class MonitorCheck
+ attr_reader :id, :monitor_id, :status, :trigger, :scheduled_for,
+ :started_at, :finished_at, :estimated_credits,
+ :reserved_credits, :actual_credits, :billing_status,
+ :summary, :target_results, :notification_status, :error,
+ :created_at, :updated_at
+
+ def initialize(data)
+ @id = data["id"]
+ @monitor_id = data["monitorId"]
+ @status = data["status"]
+ @trigger = data["trigger"]
+ @scheduled_for = data["scheduledFor"]
+ @started_at = data["startedAt"]
+ @finished_at = data["finishedAt"]
+ @estimated_credits = data["estimatedCredits"]
+ @reserved_credits = data["reservedCredits"]
+ @actual_credits = data["actualCredits"]
+ @billing_status = data["billingStatus"]
+ @summary = data["summary"] || {}
+ @target_results = data["targetResults"]
+ @notification_status = data["notificationStatus"]
+ @error = data["error"]
+ @created_at = data["createdAt"]
+ @updated_at = data["updatedAt"]
+ end
+ end
+
+ class MonitorCheckDetail < MonitorCheck
+ attr_accessor :pages, :next_url
+
+ def initialize(data)
+ super
+ @pages = data["pages"] || []
+ @next_url = data["next"]
+ end
+ end
+ end
+end
diff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rb
index 33b639d21..02eba3e87 100644
--- a/apps/ruby-sdk/lib/firecrawl/version.rb
+++ b/apps/ruby-sdk/lib/firecrawl/version.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
module Firecrawl
- VERSION = "1.3.1"
+ VERSION = "1.4.0"
end
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index 13078e92d..a0f539d34 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "2.3.1"
+version = "2.4.0"
dependencies = [
"mockito",
"reqwest",
diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.toml
index 6de8905e7..509e1dcdc 100644
--- a/apps/rust-sdk/Cargo.toml
+++ b/apps/rust-sdk/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "firecrawl"
-version = "2.3.1"
+version = "2.4.0"
edition = "2021"
license = "MIT"
homepage = "https://www.firecrawl.dev/"
diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rs
index 379ecdbc9..4639f0cdb 100644
--- a/apps/rust-sdk/src/lib.rs
+++ b/apps/rust-sdk/src/lib.rs
@@ -25,6 +25,7 @@ mod batch_scrape;
mod client;
mod crawl;
mod map;
+mod monitor;
mod parse;
mod scrape;
mod search;
@@ -36,6 +37,7 @@ pub use client::Client;
pub use crawl::*;
pub use error::FirecrawlError;
pub use map::*;
+pub use monitor::*;
pub use parse::*;
pub use scrape::*;
pub use search::*;
diff --git a/apps/rust-sdk/src/monitor.rs b/apps/rust-sdk/src/monitor.rs
new file mode 100644
index 000000000..2b7e1b493
--- /dev/null
+++ b/apps/rust-sdk/src/monitor.rs
@@ -0,0 +1,370 @@
+//! Monitor endpoint for Firecrawl API v2.
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+use crate::client::Client;
+use crate::FirecrawlError;
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct MonitorSchedule {
+ pub cron: String,
+ pub timezone: Option<String>,
+}
+
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CreateMonitorRequest {
+ pub name: String,
+ pub schedule: MonitorSchedule,
+ pub targets: Vec<Value>,
+ pub webhook: Option<Value>,
+ pub notification: Option<Value>,
+ pub retention_days: Option<u32>,
+}
+
+#[serde_with::skip_serializing_none]
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct UpdateMonitorRequest {
+ pub name: Option<String>,
+ pub status: Option<String>,
+ pub schedule: Option<MonitorSchedule>,
+ pub targets: Option<Vec<Value>>,
+ pub webhook: Option<Value>,
+ pub notification: Option<Value>,
+ pub retention_days: Option<u32>,
+}
+
+#[derive(Deserialize, Serialize, Debug, Default, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct MonitorSummary {
+ pub total_pages: u32,
+ pub same: u32,
+ pub changed: u32,
+ pub new: u32,
+ pub removed: u32,
+ pub error: u32,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct Monitor {
+ pub id: String,
+ pub name: String,
+ pub status: String,
+ pub schedule: MonitorSchedule,
+ pub next_run_at: Option<String>,
+ pub last_run_at: Option<String>,
+ pub current_check_id: Option<String>,
+ pub targets: Vec<Value>,
+ pub webhook: Option<Value>,
+ pub notification: Option<Value>,
+ pub retention_days: u32,
+ pub estimated_credits_per_month: Option<u32>,
+ pub last_check_summary: Option<MonitorSummary>,
+ pub created_at: String,
+ pub updated_at: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct MonitorCheck {
+ pub id: String,
+ pub monitor_id: String,
+ pub status: String,
+ pub trigger: String,
+ pub scheduled_for: Option<String>,
+ pub started_at: Option<String>,
+ pub finished_at: Option<String>,
+ pub estimated_credits: Option<u32>,
+ pub reserved_credits: Option<u32>,
+ pub actual_credits: Option<u32>,
+ pub billing_status: String,
+ pub summary: MonitorSummary,
+ pub target_results: Option<Value>,
+ pub notification_status: Option<Value>,
+ pub error: Option<String>,
+ pub created_at: String,
+ pub updated_at: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct MonitorCheckPage {
+ pub id: String,
+ pub target_id: String,
+ pub url: String,
+ pub status: String,
+ pub previous_scrape_id: Option<String>,
+ pub current_scrape_id: Option<String>,
+ pub status_code: Option<u16>,
+ pub error: Option<String>,
+ pub metadata: Option<Value>,
+ pub diff: Option<Value>,
+ pub created_at: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct MonitorCheckDetail {
+ #[serde(flatten)]
+ pub check: MonitorCheck,
+ pub pages: Vec<MonitorCheckPage>,
+ pub next: Option<String>,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct DataResponse<T> {
+ data: T,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct SuccessResponse {
+ success: bool,
+}
+
+fn query(limit: Option<u32>, offset: Option<u32>, status: Option<&str>) -> String {
+ let mut params = Vec::new();
+ if let Some(limit) = limit {
+ params.push(format!("limit={}", limit));
+ }
+ if let Some(offset) = offset {
+ params.push(format!("offset={}", offset));
+ }
+ if let Some(status) = status {
+ params.push(format!("status={}", status));
+ }
+ if params.is_empty() {
+ String::new()
+ } else {
+ format!("?{}", params.join("&"))
+ }
+}
+
+fn check_page_query(limit: Option<u32>, skip: Option<u32>, status: Option<&str>) -> String {
+ let mut params = Vec::new();
+ if let Some(limit) = limit {
+ params.push(format!("limit={}", limit));
+ }
+ if let Some(skip) = skip {
+ params.push(format!("skip={}", skip));
+ }
+ if let Some(status) = status {
+ params.push(format!("status={}", status));
+ }
+ if params.is_empty() {
+ String::new()
+ } else {
+ format!("?{}", params.join("&"))
+ }
+}
+
+impl Client {
+ pub async fn create_monitor(
+ &self,
+ request: CreateMonitorRequest,
+ ) -> Result<Monitor, FirecrawlError> {
+ let response = self
+ .client
+ .post(self.url("/monitor"))
+ .headers(self.prepare_headers(None))
+ .json(&request)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Creating monitor".to_string(), e))?;
+
+ let response: DataResponse<Monitor> =
+ self.handle_response(response, "create monitor").await?;
+ Ok(response.data)
+ }
+
+ pub async fn list_monitors(
+ &self,
+ limit: Option<u32>,
+ offset: Option<u32>,
+ ) -> Result<Vec<Monitor>, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/monitor{}", query(limit, offset, None))))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Listing monitors".to_string(), e))?;
+
+ let response: DataResponse<Vec<Monitor>> =
+ self.handle_response(response, "list monitors").await?;
+ Ok(response.data)
+ }
+
+ pub async fn get_monitor(
+ &self,
+ monitor_id: impl AsRef<str>,
+ ) -> Result<Monitor, FirecrawlError> {
+ let response = self
+ .client
+ .get(self.url(&format!("/monitor/{}", monitor_id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Getting monitor".to_string(), e))?;
+
+ let response: DataResponse<Monitor> = self.handle_response(response, "get monitor").await?;
+ Ok(response.data)
+ }
+
+ pub async fn update_monitor(
+ &self,
+ monitor_id: impl AsRef<str>,
+ request: UpdateMonitorRequest,
+ ) -> Result<Monitor, FirecrawlError> {
+ let response = self
+ .client
+ .patch(self.url(&format!("/monitor/{}", monitor_id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .json(&request)
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Updating monitor".to_string(), e))?;
+
+ let response: DataResponse<Monitor> =
+ self.handle_response(response, "update monitor").await?;
+ Ok(response.data)
+ }
+
+ pub async fn delete_monitor(
+ &self,
+ monitor_id: impl AsRef<str>,
+ ) -> Result<bool, FirecrawlError> {
+ let response = self
+ .client
+ .delete(self.url(&format!("/monitor/{}", monitor_id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Deleting monitor".to_string(), e))?;
+
+ let response: SuccessResponse = self.handle_response(response, "delete monitor").await?;
+ Ok(response.success)
+ }
+
+ pub async fn run_monitor(
+ &self,
+ monitor_id: impl AsRef<str>,
+ ) -> Result<MonitorCheck, FirecrawlError> {
+ let response = self
+ .client
+ .post(self.url(&format!("/monitor/{}/run", monitor_id.as_ref())))
+ .headers(self.prepare_headers(None))
+ .json(&serde_json::json!({}))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Running monitor".to_string(), e))?;
+
+ let response: DataResponse<MonitorCheck> =
+ self.handle_response(response, "run monitor").await?;
+ Ok(response.data)
+ }
+
+ pub async fn list_monitor_checks(
+ &self,
+ monitor_id: impl AsRef<str>,
+ limit: Option<u32>,
+ offset: Option<u32>,
+ ) -> Result<Vec<MonitorCheck>, FirecrawlError> {
+ let path = format!(
+ "/monitor/{}/checks{}",
+ monitor_id.as_ref(),
+ query(limit, offset, None)
+ );
+ let response = self
+ .client
+ .get(self.url(&path))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Listing monitor checks".to_string(), e))?;
+
+ let response: DataResponse<Vec<MonitorCheck>> = self
+ .handle_response(response, "list monitor checks")
+ .await?;
+ Ok(response.data)
+ }
+
+ pub async fn get_monitor_check(
+ &self,
+ monitor_id: impl AsRef<str>,
+ check_id: impl AsRef<str>,
+ limit: Option<u32>,
+ skip: Option<u32>,
+ status: Option<&str>,
+ ) -> Result<MonitorCheckDetail, FirecrawlError> {
+ let path = format!(
+ "/monitor/{}/checks/{}{}",
+ monitor_id.as_ref(),
+ check_id.as_ref(),
+ check_page_query(limit, skip, status)
+ );
+ let response = self
+ .client
+ .get(self.url(&path))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Getting monitor check".to_string(), e))?;
+
+ let response: DataResponse<MonitorCheckDetail> =
+ self.handle_response(response, "get monitor check").await?;
+ let mut check = response.data;
+
+ while let Some(next) = check.next.clone() {
+ let response = self
+ .client
+ .get(next)
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| {
+ FirecrawlError::HttpError("Getting monitor check page".to_string(), e)
+ })?;
+ let response: DataResponse<MonitorCheckDetail> = self
+ .handle_response(response, "get monitor check page")
+ .await?;
+ check.pages.extend(response.data.pages);
+ check.next = response.data.next;
+ }
+
+ Ok(check)
+ }
+
+ pub async fn get_monitor_check_page(
+ &self,
+ monitor_id: impl AsRef<str>,
+ check_id: impl AsRef<str>,
+ limit: Option<u32>,
+ skip: Option<u32>,
+ status: Option<&str>,
+ ) -> Result<MonitorCheckDetail, FirecrawlError> {
+ let path = format!(
+ "/monitor/{}/checks/{}{}",
+ monitor_id.as_ref(),
+ check_id.as_ref(),
+ check_page_query(limit, skip, status)
+ );
+ let response = self
+ .client
+ .get(self.url(&path))
+ .headers(self.prepare_headers(None))
+ .send()
+ .await
+ .map_err(|e| FirecrawlError::HttpError("Getting monitor check".to_string(), e))?;
+
+ let response: DataResponse<MonitorCheckDetail> =
+ self.handle_response(response, "get monitor check").await?;
+ Ok(response.data)
+ }
+}