Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add monitor API orchestration (ENG-4857) (#3470)
Gergő Móriczbc3c57adbb26
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex 4e2306633..4d0e461d0 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -23,6 +23,7 @@ export const TEST_PRODUCTION = !TEST_SELF_HOST;// TODO: do we want to run AI tests when users run this command locally? It may lead to increased spending for them, depending on configurationexport const HAS_AI = !!(config.OPENAI_API_KEY || config.OLLAMA_BASE_URL);+export const HAS_FIREWORKS = !!process.env.FIREWORKS_API_KEY;export const HAS_FIRE_ENGINE = !!config.FIRE_ENGINE_BETA_URL;export const HAS_PLAYWRIGHT = !!config.PLAYWRIGHT_MICROSERVICE_URL;export const HAS_PROXY = !!config.PROXY_SERVER;diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 88a8228a8..53c70e4db 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -437,7 +437,7 @@ describe("Crawl tests", () => {async () => {const res = await crawl({- url: base,+ url: new URL("/blog", base).href,prompt:"Crawl everything including external links and subdomains",// Explicit options that should override the promptdiff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 840a2c7c7..1e5583f71 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -85,6 +85,96 @@ export async function scrapeWithFailure(return raw.body;}+// =========================================+// Monitor API+// =========================================++export type MonitorCreateInput = {+ name: string;+ schedule: { cron: string; timezone?: string };+ webhook?: { url: string; headers?: Record<string, string> };+ notification?: {+ email?: {+ enabled?: boolean;+ recipients?: string[];+ includeDiffs?: boolean;+ };+ };+ targets: Array<+ | {+ type: "scrape";+ urls: string[];+ scrapeOptions?: Record<string, unknown>;+ }+ | {+ type: "crawl";+ url: string;+ crawlOptions?: Record<string, unknown>;+ scrapeOptions?: Record<string, unknown>;+ }+ >;+ retentionDays?: number;+};++export async function monitorCreateRaw(+ body: MonitorCreateInput,+ identity: Identity,+) {+ return await request(TEST_API_URL)+ .post("/v2/monitor")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body);+}++export async function monitorListRaw(identity: Identity) {+ return await request(TEST_API_URL)+ .get("/v2/monitor")+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++export async function monitorGetRaw(id: string, identity: Identity) {+ return await request(TEST_API_URL)+ .get(`/v2/monitor/${id}`)+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++export async function monitorPatchRaw(+ id: string,+ body: Partial<MonitorCreateInput> & { status?: "active" | "paused" },+ identity: Identity,+) {+ return await request(TEST_API_URL)+ .patch(`/v2/monitor/${id}`)+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send(body);+}++export async function monitorDeleteRaw(id: string, identity: Identity) {+ return await request(TEST_API_URL)+ .delete(`/v2/monitor/${id}`)+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++export async function monitorRunRaw(id: string, identity: Identity) {+ return await request(TEST_API_URL)+ .post(`/v2/monitor/${id}/run`)+ .set("Authorization", `Bearer ${identity.apiKey}`);+}++export async function monitorCheckRaw(+ monitorId: string,+ checkId: string,+ identity: Identity,+ query?: Record<string, string | number>,+) {+ const req = request(TEST_API_URL)+ .get(`/v2/monitor/${monitorId}/checks/${checkId}`)+ .set("Authorization", `Bearer ${identity.apiKey}`);+ return query ? req.query(query) : req;+}+export async function parseRaw(body: {options?: Omit<ParseRequestInput, "file">;diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.tsnew file mode 100644index 000000000..9c677507a--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts@@ -0,0 +1,142 @@+import {+ createTestIdUrl,+ describeIf,+ ALLOW_TEST_SUITE_WEBSITE,+ TEST_SELF_HOST,+} from "../lib";+import {+ idmux,+ Identity,+ monitorCheckRaw,+ monitorCreateRaw,+ monitorDeleteRaw,+ monitorGetRaw,+ monitorListRaw,+ monitorPatchRaw,+ monitorRunRaw,+ scrapeTimeout,+} from "./lib";++describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "monitor",+ concurrency: 20,+ credits: 1000000,+ });+ }, 10000);++ it("creates, lists, gets, pauses, and deletes a monitor", async () => {+ const create = await monitorCreateRaw(+ {+ name: "snips monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl(), createTestIdUrl()],+ scrapeOptions: { formats: ["markdown"] },+ },+ ],+ notification: { email: { enabled: false } },+ },+ identity,+ );++ expect(create.statusCode).toBe(200);+ expect(create.body.success).toBe(true);+ expect(create.body.data.id).toEqual(expect.any(String));+ expect(create.body.data.targets[0].id).toEqual(expect.any(String));++ const id = create.body.data.id;+ const list = await monitorListRaw(identity);+ expect(list.statusCode).toBe(200);+ expect(list.body.data.some((x: any) => x.id === id)).toBe(true);++ const get = await monitorGetRaw(id, identity);+ expect(get.statusCode).toBe(200);+ expect(get.body.data.id).toBe(id);++ const patch = await monitorPatchRaw(id, { status: "paused" }, identity);+ expect(patch.statusCode).toBe(200);+ expect(patch.body.data.status).toBe("paused");++ const del = await monitorDeleteRaw(id, identity);+ expect(del.statusCode).toBe(200);+ expect(del.body.success).toBe(true);+ });++ it("rejects cron schedules under 15 minutes", async () => {+ const response = await monitorCreateRaw(+ {+ name: "too frequent",+ schedule: { cron: "*/5 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl()],+ },+ ],+ },+ identity,+ );++ expect(response.statusCode).toBe(400);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain("15 minutes");+ });++ it(+ "runs a manual scrape monitor check",+ async () => {+ const create = await monitorCreateRaw(+ {+ name: "manual monitor",+ schedule: { cron: "*/30 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [createTestIdUrl(), createTestIdUrl()],+ scrapeOptions: { formats: ["markdown"] },+ },+ ],+ },+ identity,+ );+ expect(create.statusCode).toBe(200);++ const monitorId = create.body.data.id;+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ const checkId = run.body.id;++ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, checkId, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (["completed", "partial", "failed"].includes(check.status)) break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }++ expect(["completed", "partial"]).toContain(check.status);+ expect(check.summary.totalPages).toBeGreaterThanOrEqual(2);+ expect(check.pages.length).toBeGreaterThanOrEqual(1);+ expect(check.next).toBeUndefined();++ const firstPage = await monitorCheckRaw(monitorId, checkId, identity, {+ limit: 1,+ });+ expect(firstPage.statusCode).toBe(200);+ expect(firstPage.body.next).toContain("skip=1");+ expect(firstPage.body.next).toContain("limit=1");+ expect(firstPage.body.data.next).toBe(firstPage.body.next);+ expect(firstPage.body.data.pages).toHaveLength(1);++ await monitorDeleteRaw(monitorId, identity);+ },+ 2 * scrapeTimeout,+ );+});diff --git a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts b/apps/api/src/__tests__/snips/v2/scrape-query.test.tsindex 41cbd5b4d..b088edc1e 100644--- a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-query.test.ts@@ -1,4 +1,4 @@-import { concurrentIf, HAS_AI, TEST_PRODUCTION } from "../lib";+import { concurrentIf, HAS_AI, HAS_FIREWORKS, TEST_PRODUCTION } from "../lib";import {scrape,scrapeRaw,@@ -61,7 +61,7 @@ describe("Query format", () => {scrapeTimeout,);- concurrentIf(TEST_PRODUCTION || HAS_AI)(+ concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)("returns a direct quote answer when query mode is directQuote",async () => {const response = await scrape(diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 3ab18a905..7aa29e744 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -21,6 +21,11 @@ import {SearchRequestInput,toV2CrawlerOptions,} from "../../../controllers/v2/types";+import {+ createMonitorSchema,+ updateMonitorSchema,+} from "../../../services/monitoring/types";+import { getNextMonitorRunAt } from "../../../services/monitoring/cron";describe("V2 Types Validation", () => {describe("scrapeRequestSchema", () => {@@ -1164,6 +1169,87 @@ describe("V2 Types Validation", () => {});});+ describe("monitor schedules", () => {+ it("should accept natural language schedule text", () => {+ const result = createMonitorSchema.parse({+ name: "Blog monitor",+ schedule: {+ text: "every 30 minutes",+ },+ targets: [+ {+ type: "scrape",+ urls: ["https://example.com"],+ },+ ],+ });++ expect(result.schedule).toEqual({+ cron: "*/30 * * * *",+ timezone: "UTC",+ });+ });++ it("should accept a natural language start minute", () => {+ const result = updateMonitorSchema.parse({+ schedule: {+ text: "every 15 minutes starting at :07",+ },+ });++ expect(result.schedule).toEqual({+ cron: "7-59/15 * * * *",+ timezone: "UTC",+ });+ });++ it("should reject ambiguous schedule definitions", () => {+ expect(() =>+ updateMonitorSchema.parse({+ schedule: {+ cron: "*/30 * * * *",+ text: "every 30 minutes",+ },+ }),+ ).toThrow("Schedule must include either cron or text, not both");+ });++ it("should calculate next runs in the configured timezone", () => {+ const next = getNextMonitorRunAt(+ "0 9 * * *",+ new Date("2026-01-01T13:00:00.000Z"),+ "America/New_York",+ );++ expect(next.toISOString()).toBe("2026-01-01T14:00:00.000Z");+ });++ it("should accept monitor webhook event filters", () => {+ const result = updateMonitorSchema.parse({+ webhook: {+ url: "https://example.com/webhook",+ events: ["monitor.page", "monitor.check.completed"],+ },+ });++ expect(result.webhook?.events).toEqual([+ "monitor.page",+ "monitor.check.completed",+ ]);+ });++ it("should reject non-monitor webhook event filters", () => {+ expect(() =>+ updateMonitorSchema.parse({+ webhook: {+ url: "https://example.com/webhook",+ events: ["completed"],+ },+ }),+ ).toThrow();+ });+ });+describe("Edge cases", () => {it("should handle URL without protocol (should add http://)", () => {const input = {diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsnew file mode 100644index 000000000..914b43c85--- /dev/null+++ b/apps/api/src/controllers/v2/monitor.ts@@ -0,0 +1,353 @@+import { Response } from "express";+import { z } from "zod";+import { RequestWithAuth } from "./types";+import { getScrapeZDR } from "../../lib/zdr-helpers";+import { getMonitorDiffArtifact } from "../../lib/gcs-monitoring";+import {+ createMonitorSchema,+ listMonitorChecksQuerySchema,+ listMonitorsQuerySchema,+ monitorCheckDetailQuerySchema,+ updateMonitorSchema,+} from "../../services/monitoring/types";+import {+ createMonitor,+ createMonitorCheck,+ countMonitorCheckPages,+ deleteMonitor,+ estimateMonitorCreditsPerRun,+ getMonitor,+ getMonitorCheck,+ getMonitorForUpdate,+ listMonitorCheckPages,+ listMonitorChecks,+ listMonitors,+ updateMonitor,+} from "../../services/monitoring/store";+import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";+import {+ estimateRunsPerMonth,+ validateMonitorCron,+} from "../../services/monitoring/cron";++const monitorParamsSchema = z.strictObject({+ monitorId: z.uuid(),+});++const monitorCheckParamsSchema = monitorParamsSchema.extend({+ checkId: z.uuid(),+});++function rejectZdr(+ req: RequestWithAuth<any, any, any>,+ res: Response,+): boolean {+ if (getScrapeZDR(req.acuc?.flags) === "forced") {+ res.status(400).json({+ success: false,+ error:+ "Monitoring requires retained snapshots and diffs, and is not supported for zero data retention teams.",+ });+ return true;+ }+ return false;+}++function serializeMonitor(monitor: any) {+ return {+ id: monitor.id,+ name: monitor.name,+ status: monitor.status,+ schedule: {+ cron: monitor.schedule_cron,+ timezone: monitor.schedule_timezone,+ },+ nextRunAt: monitor.next_run_at,+ lastRunAt: monitor.last_run_at,+ currentCheckId: monitor.current_check_id,+ targets: monitor.targets,+ webhook: monitor.webhook,+ notification: monitor.notification,+ retentionDays: monitor.retention_days,+ estimatedCreditsPerMonth: monitor.estimated_credits_per_month,+ lastCheckSummary: monitor.last_check_summary,+ createdAt: monitor.created_at,+ updatedAt: monitor.updated_at,+ };+}++function serializeCheck(check: any) {+ return {+ id: check.id,+ monitorId: check.monitor_id,+ status: check.status,+ trigger: check.trigger,+ scheduledFor: check.scheduled_for,+ startedAt: check.started_at,+ finishedAt: check.finished_at,+ estimatedCredits: check.estimated_credits,+ reservedCredits: check.reserved_credits,+ actualCredits: check.actual_credits,+ billingStatus: check.billing_status,+ summary: {+ totalPages: check.total_pages,+ same: check.same_count,+ changed: check.changed_count,+ new: check.new_count,+ removed: check.removed_count,+ error: check.error_count,+ },+ targetResults: check.target_results,+ notificationStatus: check.notification_status,+ error: check.error,+ createdAt: check.created_at,+ updatedAt: check.updated_at,+ };+}++export async function createMonitorController(+ req: RequestWithAuth<{}, any, unknown>,+ res: Response,+) {+ if (rejectZdr(req, res)) return;++ const input = createMonitorSchema.parse(req.body);+ let schedule;+ try {+ schedule = validateMonitorCron(+ input.schedule.cron,+ input.schedule.timezone,+ );+ } catch (error) {+ return res.status(400).json({+ success: false,+ error: error instanceof Error ? error.message : String(error),+ });+ }+ const monitor = await createMonitor({+ teamId: req.auth.team_id,+ input,+ nextRunAt: schedule.nextRunAt,+ intervalMs: schedule.intervalMs,+ });++ res.status(200).json({+ success: true,+ data: serializeMonitor(monitor),+ });+}++export async function listMonitorsController(+ req: RequestWithAuth<{}, any, unknown>,+ res: Response,+) {+ const query = listMonitorsQuerySchema.parse(req.query);+ const monitors = await listMonitors({+ teamId: req.auth.team_id,+ limit: query.limit,+ offset: query.offset,+ });++ res.status(200).json({+ success: true,+ data: monitors.map(serializeMonitor),+ });+}++export async function getMonitorController(+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,+ res: Response,+) {+ const { monitorId } = monitorParamsSchema.parse(req.params);+ const monitor = await getMonitor(req.auth.team_id, monitorId);+ if (!monitor) {+ return res.status(404).json({ success: false, error: "Monitor not found" });+ }++ res.status(200).json({+ success: true,+ data: serializeMonitor(monitor),+ });+}++export async function updateMonitorController(+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,+ res: Response,+) {+ if (rejectZdr(req, res)) return;++ const { monitorId } = monitorParamsSchema.parse(req.params);+ const existing = await getMonitorForUpdate(req.auth.team_id, monitorId);+ if (!existing) {+ return res.status(404).json({ success: false, error: "Monitor not found" });+ }++ const input = updateMonitorSchema.parse(req.body);+ const cron = input.schedule?.cron ?? existing.schedule_cron;+ const timezone = input.schedule?.timezone ?? existing.schedule_timezone;+ let schedule;+ try {+ schedule = validateMonitorCron(cron, timezone);+ } catch (error) {+ return res.status(400).json({+ success: false,+ error: error instanceof Error ? error.message : String(error),+ });+ }+ const monitor = await updateMonitor({+ teamId: req.auth.team_id,+ monitorId,+ input,+ nextRunAt: input.schedule ? schedule.nextRunAt : undefined,+ intervalMs:+ input.schedule || input.targets ? schedule.intervalMs : undefined,+ });++ res.status(200).json({+ success: true,+ data: serializeMonitor(monitor),+ });+}++export async function deleteMonitorController(+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,+ res: Response,+) {+ const { monitorId } = monitorParamsSchema.parse(req.params);+ const deleted = await deleteMonitor({+ teamId: req.auth.team_id,+ monitorId,+ });+ if (!deleted) {+ return res.status(404).json({ success: false, error: "Monitor not found" });+ }++ res.status(200).json({ success: true });+}++export async function runMonitorController(+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,+ res: Response,+) {+ if (rejectZdr(req, res)) return;++ const { monitorId } = monitorParamsSchema.parse(req.params);+ const monitor = await getMonitorForUpdate(req.auth.team_id, monitorId);+ if (!monitor) {+ return res.status(404).json({ success: false, error: "Monitor not found" });+ }+ if (monitor.current_check_id) {+ return res.status(409).json({+ success: false,+ error: "Monitor check is already running.",+ checkId: monitor.current_check_id,+ });+ }++ const check = await createMonitorCheck({+ monitor,+ trigger: "manual",+ });+ await enqueueMonitorCheck({+ monitorId: monitor.id,+ checkId: check.id,+ teamId: monitor.team_id,+ });++ res.status(200).json({+ success: true,+ id: check.id,+ data: serializeCheck(check),+ });+}++export async function listMonitorChecksController(+ req: RequestWithAuth<{ monitorId: string }, any, unknown>,+ res: Response,+) {+ const { monitorId } = monitorParamsSchema.parse(req.params);+ const monitor = await getMonitor(req.auth.team_id, monitorId);+ if (!monitor) {+ return res.status(404).json({ success: false, error: "Monitor not found" });+ }++ const query = listMonitorChecksQuerySchema.parse(req.query);+ const checks = await listMonitorChecks({+ teamId: req.auth.team_id,+ monitorId,+ limit: query.limit,+ offset: query.offset,+ status: query.status,+ });++ res.status(200).json({+ success: true,+ data: checks.map(serializeCheck),+ });+}++export async function getMonitorCheckController(+ req: RequestWithAuth<{ monitorId: string; checkId: string }, any, unknown>,+ res: Response,+) {+ const { monitorId, checkId } = monitorCheckParamsSchema.parse(req.params);+ const query = monitorCheckDetailQuerySchema.parse(req.query);+ const skip = query.skip;+ const check = await getMonitorCheck(req.auth.team_id, monitorId, checkId);+ if (!check) {+ return res.status(404).json({ success: false, error: "Check not found" });+ }++ const [pages, totalPagesForFilter] = await Promise.all([+ listMonitorCheckPages({+ teamId: req.auth.team_id,+ monitorId,+ checkId,+ limit: query.limit,+ skip,+ status: query.status,+ }),+ countMonitorCheckPages({+ checkId,+ status: query.status,+ }),+ ]);++ const pagesWithDiffs = await Promise.all(+ pages.map(async page => ({+ id: page.id,+ targetId: page.target_id,+ url: page.url,+ status: page.status,+ previousScrapeId: page.previous_scrape_id,+ currentScrapeId: page.current_scrape_id,+ statusCode: page.status_code,+ error: page.error,+ metadata: page.metadata,+ diff: await getMonitorDiffArtifact(page.diff_gcs_key),+ createdAt: page.created_at,+ })),+ );+ const nextSkip = skip + pagesWithDiffs.length;+ const next = (() => {+ if (totalPagesForFilter <= nextSkip) return undefined;+ const url = new URL(+ `/v2/monitor/${monitorId}/checks/${checkId}`,+ `${req.protocol}://${req.get("host")}`,+ );+ url.searchParams.set("skip", String(nextSkip));+ url.searchParams.set("limit", String(query.limit));+ if (query.status) url.searchParams.set("status", query.status);+ return url.toString();+ })();++ res.status(200).json({+ success: true,+ next,+ data: {+ ...serializeCheck(check),+ pages: pagesWithDiffs,+ next,+ },+ });+}diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 1deef110a..51e5c1822 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -1,5 +1,6 @@import { config } from "./config";import { type ChildProcess, spawn } from "child_process";+import { existsSync } from "fs";import * as net from "net";import { basename, join } from "path";import { HTML_TO_MARKDOWN_PATH } from "./natives";@@ -19,9 +20,15 @@ let nuqRabbitMQContainer: {containerRuntime: string;} | null = null;-// Get the monorepo root (apps/api/dist/src -> ../../../..)+// Get the monorepo root for both tsx source execution and compiled dist execution.// __dirname is available in CommonJS (which this compiles to)-const MONOREPO_ROOT = join(__dirname, "..", "..", "..", "..");+const SOURCE_MONOREPO_ROOT = join(__dirname, "..", "..", "..");+const DIST_MONOREPO_ROOT = join(__dirname, "..", "..", "..", "..");+const MONOREPO_ROOT = existsSync(+ join(SOURCE_MONOREPO_ROOT, "apps", "nuq-postgres"),+)+ ? SOURCE_MONOREPO_ROOT+ : DIST_MONOREPO_ROOT;const NUQ_POSTGRES_PATH = join(MONOREPO_ROOT, "apps", "nuq-postgres");interface ProcessResult {diff --git a/apps/api/src/lib/gcs-monitoring.ts b/apps/api/src/lib/gcs-monitoring.tsnew file mode 100644index 000000000..36b3e9e12--- /dev/null+++ b/apps/api/src/lib/gcs-monitoring.ts@@ -0,0 +1,64 @@+import { config } from "../config";+import { storage } from "./gcs-jobs";++type MonitorDiffArtifact = {+ url: string;+ previousScrapeId: string | null;+ currentScrapeId: string | null;+ text: string;+ json: unknown;+ generatedAt: string;+};++const contentType = "application/json";++export function monitorDiffGcsKey(params: {+ teamId: string;+ monitorId: string;+ checkId: string;+ pageId: string;+}): string {+ return `monitors/${params.teamId}/${params.monitorId}/${params.checkId}/${params.pageId}.diff.json`;+}++export async function saveMonitorDiffArtifact(+ key: string,+ artifact: MonitorDiffArtifact,+): Promise<{ textBytes: number; jsonBytes: number }> {+ const payload = JSON.stringify(artifact);+ if (!config.GCS_BUCKET_NAME) {+ return {+ textBytes: Buffer.byteLength(artifact.text),+ jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),+ };+ }++ const bucket = storage.bucket(config.GCS_BUCKET_NAME);+ await bucket.file(key).save(payload, {+ contentType,+ resumable: false,+ });++ return {+ textBytes: Buffer.byteLength(artifact.text),+ jsonBytes: Buffer.byteLength(JSON.stringify(artifact.json ?? null)),+ };+}++export async function getMonitorDiffArtifact(+ key: string | null | undefined,+): Promise<MonitorDiffArtifact | null> {+ if (!key || !config.GCS_BUCKET_NAME) return null;++ const bucket = storage.bucket(config.GCS_BUCKET_NAME);+ try {+ const [contents] = await bucket.file(key).download();+ return JSON.parse(contents.toString()) as MonitorDiffArtifact;+ } catch (error) {+ const maybeGcsError = error as { code?: number; statusCode?: number };+ if (maybeGcsError.code === 404 || maybeGcsError.statusCode === 404) {+ return null;+ }+ throw error;+ }+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex d84e30221..407e1f2de 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -65,6 +65,16 @@ import {scrapeInteractController,scrapeStopInteractiveBrowserController,} from "../controllers/v2/scrape-browser";+import {+ createMonitorController,+ deleteMonitorController,+ getMonitorCheckController,+ getMonitorController,+ listMonitorChecksController,+ listMonitorsController,+ runMonitorController,+ updateMonitorController,+} from "../controllers/v2/monitor";expressWs(express());@@ -448,6 +458,63 @@ v2Router.get(wrap(activityController),);+v2Router.post(+ "/monitor",+ authMiddleware(RateLimiterMode.Crawl),+ countryCheck,+ checkCreditsMiddleware(1),+ blocklistMiddleware,+ wrap(createMonitorController),+);++v2Router.get(+ "/monitor",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(listMonitorsController),+);++v2Router.get(+ "/monitor/:monitorId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(getMonitorController),+);++v2Router.patch(+ "/monitor/:monitorId",+ authMiddleware(RateLimiterMode.Crawl),+ countryCheck,+ checkCreditsMiddleware(1),+ blocklistMiddleware,+ wrap(updateMonitorController),+);++v2Router.delete(+ "/monitor/:monitorId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(deleteMonitorController),+);++v2Router.post(+ "/monitor/:monitorId/run",+ authMiddleware(RateLimiterMode.Crawl),+ countryCheck,+ checkCreditsMiddleware(1),+ blocklistMiddleware,+ wrap(runMonitorController),+);++v2Router.get(+ "/monitor/:monitorId/checks",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(listMonitorChecksController),+);++v2Router.get(+ "/monitor/:monitorId/checks/:checkId",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(getMonitorCheckController),+);+v2Router.post("/browser",authMiddleware(RateLimiterMode.Browser),diff --git a/apps/api/src/services/billing/types.ts b/apps/api/src/services/billing/types.tsindex e39dfb449..87ae1ec54 100644--- a/apps/api/src/services/billing/types.ts+++ b/apps/api/src/services/billing/types.ts@@ -9,6 +9,7 @@ export type BillingEndpoint =| "interact"| "llms_txt"| "map"+ | "monitor"| "parse"| "scrape"| "search";diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 1a6a11e01..5241a6d37 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -222,6 +222,8 @@ export type LoggedScrape = {skipNuq: boolean;zeroDataRetention: boolean;is_parse?: boolean;+ monitor_id?: string | null;+ monitor_check_id?: string | null;};export async function logScrape(scrape: LoggedScrape, force: boolean = false) {@@ -259,6 +261,12 @@ export async function logScrape(scrape: LoggedScrape, force: boolean = false) {? null: (scrape.pdf_num_pages ?? null),credits_cost: scrape.credits_cost,+ ...(scrape.is_parse+ ? {}+ : {+ monitor_id: scrape.monitor_id ?? null,+ monitor_check_id: scrape.monitor_check_id ?? null,+ }),},force,logger,@@ -322,6 +330,8 @@ type LoggedCrawl = {credits_cost: number;zeroDataRetention: boolean;cancelled: boolean;+ monitor_id?: string | null;+ monitor_check_id?: string | null;};export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {@@ -350,6 +360,8 @@ export async function logCrawl(crawl: LoggedCrawl, force: boolean = false) {num_docs: crawl.num_docs,credits_cost: crawl.credits_cost,cancelled: crawl.cancelled,+ monitor_id: crawl.monitor_id ?? null,+ monitor_check_id: crawl.monitor_check_id ?? null,},force,logger,diff --git a/apps/api/src/services/monitoring/cron.ts b/apps/api/src/services/monitoring/cron.tsnew file mode 100644index 000000000..cc4313a16--- /dev/null+++ b/apps/api/src/services/monitoring/cron.ts@@ -0,0 +1,247 @@+const MIN_MONITOR_INTERVAL_MS = 15 * 60 * 1000;+const SEARCH_LIMIT_MINUTES = 366 * 24 * 60;++type CronField = Set<number>;+type CronSpec = ReturnType<typeof parseCron>;++const WEEKDAY_TO_NUMBER: Record<string, number> = {+ sun: 0,+ mon: 1,+ tue: 2,+ wed: 3,+ thu: 4,+ fri: 5,+ sat: 6,+};++function parseMinuteStart(value: string | undefined): number {+ if (value === undefined) return 0;+ const minute = Number(value);+ if (!Number.isInteger(minute) || minute < 0 || minute > 59) {+ throw new Error("Schedule start minute must be between 0 and 59");+ }+ return minute;+}++function assertDivides(value: number, divisor: number, unit: string) {+ if (divisor % value !== 0) {+ throw new Error(`${unit} interval must divide evenly into ${divisor}`);+ }+}++export function parseMonitorScheduleText(input: string): string {+ const text = input+ .trim()+ .toLowerCase()+ .replace(/[.,]/g, "")+ .replace(/\s+/g, " ");++ const minuteMatch = text.match(+ /^every (\d+) (?:minutes?|mins?)(?: (?:starting|start)(?: at| on)? :?(\d{1,2}))?$/,+ );+ if (minuteMatch) {+ const interval = Number(minuteMatch[1]);+ if (!Number.isInteger(interval) || interval <= 0 || interval > 60) {+ throw new Error("Minute interval must be between 1 and 60");+ }+ assertDivides(interval, 60, "Minute");+ const startMinute = parseMinuteStart(minuteMatch[2]);+ return startMinute === 0+ ? `*/${interval} * * * *`+ : `${startMinute}-59/${interval} * * * *`;+ }++ const hourlyMatch = text.match(+ /^(?:hourly|every hour)(?: (?:at|starting at) :?(\d{1,2}))?$/,+ );+ if (hourlyMatch) {+ return `${parseMinuteStart(hourlyMatch[1])} * * * *`;+ }++ const hourMatch = text.match(/^every (\d+) (?:hours?|hrs?)$/);+ if (hourMatch) {+ const interval = Number(hourMatch[1]);+ if (!Number.isInteger(interval) || interval <= 0 || interval > 24) {+ throw new Error("Hour interval must be between 1 and 24");+ }+ assertDivides(interval, 24, "Hour");+ return `0 */${interval} * * *`;+ }++ const dailyMatch = text.match(+ /^(?:daily|every day)(?: at (\d{1,2})(?::(\d{2}))?)?$/,+ );+ if (dailyMatch) {+ const hour = dailyMatch[1] === undefined ? 0 : Number(dailyMatch[1]);+ const minute = dailyMatch[2] === undefined ? 0 : Number(dailyMatch[2]);+ if (!Number.isInteger(hour) || hour < 0 || hour > 23) {+ throw new Error("Daily schedule hour must be between 0 and 23");+ }+ if (!Number.isInteger(minute) || minute < 0 || minute > 59) {+ throw new Error("Daily schedule minute must be between 0 and 59");+ }+ return `${minute} ${hour} * * *`;+ }++ if (text === "weekly" || text === "every week") {+ return "0 0 * * 0";+ }++ throw new Error(+ "Unsupported schedule text. Try phrases like 'every 30 minutes', 'hourly', or 'daily at 9:00'",+ );+}++function parseField(field: string, min: number, max: number): CronField {+ const values = new Set<number>();+ for (const part of field.split(",")) {+ const [rangePart, stepPart] = part.split("/");+ const step = stepPart === undefined ? 1 : Number(stepPart);+ if (!Number.isInteger(step) || step <= 0) {+ throw new Error("Invalid cron step");+ }++ let start: number;+ let end: number;+ if (rangePart === "*") {+ start = min;+ end = max;+ } else if (rangePart.includes("-")) {+ const [a, b] = rangePart.split("-").map(Number);+ start = a;+ end = b;+ } else {+ start = Number(rangePart);+ end = start;+ }++ if (+ !Number.isInteger(start) ||+ !Number.isInteger(end) ||+ start < min ||+ end > max ||+ start > end+ ) {+ throw new Error("Invalid cron field");+ }++ for (let value = start; value <= end; value += step) {+ values.add(value);+ }+ }+ return values;+}++function parseDayOfWeek(field: string): CronField {+ const values = parseField(field, 0, 7);+ if (values.has(7)) {+ values.add(0);+ values.delete(7);+ }+ return values;+}++function parseCron(cron: string) {+ const parts = cron.trim().split(/\s+/);+ if (parts.length !== 5) {+ throw new Error("Cron expression must contain five fields");+ }++ return {+ minutes: parseField(parts[0], 0, 59),+ hours: parseField(parts[1], 0, 23),+ daysOfMonth: parseField(parts[2], 1, 31),+ months: parseField(parts[3], 1, 12),+ daysOfWeek: parseDayOfWeek(parts[4]),+ };+}++function validateTimeZone(timeZone: string): void {+ try {+ new Intl.DateTimeFormat("en-US", { timeZone }).format(new Date());+ } catch {+ throw new Error(`Invalid monitor schedule timezone: ${timeZone}`);+ }+}++function getZonedParts(date: Date, timeZone: string) {+ const parts = new Intl.DateTimeFormat("en-US", {+ timeZone,+ hourCycle: "h23",+ minute: "2-digit",+ hour: "2-digit",+ day: "2-digit",+ month: "2-digit",+ weekday: "short",+ }).formatToParts(date);++ const values = Object.fromEntries(+ parts+ .filter(part => part.type !== "literal")+ .map(part => [part.type, part.value]),+ );++ return {+ minutes: Number(values.minute),+ hours: Number(values.hour),+ daysOfMonth: Number(values.day),+ months: Number(values.month),+ daysOfWeek: WEEKDAY_TO_NUMBER[String(values.weekday).toLowerCase()],+ };+}++function matches(date: Date, cron: CronSpec, timeZone: string): boolean {+ const zoned = getZonedParts(date, timeZone);+ return (+ cron.minutes.has(zoned.minutes) &&+ cron.hours.has(zoned.hours) &&+ cron.daysOfMonth.has(zoned.daysOfMonth) &&+ cron.months.has(zoned.months) &&+ cron.daysOfWeek.has(zoned.daysOfWeek)+ );+}++export function getNextMonitorRunAt(+ cronExpression: string,+ from = new Date(),+ timeZone = "UTC",+): Date {+ validateTimeZone(timeZone);+ const cron = parseCron(cronExpression);+ const candidate = new Date(from);+ candidate.setUTCSeconds(0, 0);+ candidate.setUTCMinutes(candidate.getUTCMinutes() + 1);++ for (let i = 0; i < SEARCH_LIMIT_MINUTES; i++) {+ if (matches(candidate, cron, timeZone)) {+ return new Date(candidate);+ }+ candidate.setUTCMinutes(candidate.getUTCMinutes() + 1);+ }++ throw new Error("Cron expression did not produce a run within one year");+}++export function validateMonitorCron(+ cronExpression: string,+ timeZone = "UTC",+): {+ nextRunAt: Date;+ intervalMs: number;+} {+ const nextRunAt = getNextMonitorRunAt(cronExpression, new Date(), timeZone);+ const secondRunAt = getNextMonitorRunAt(cronExpression, nextRunAt, timeZone);+ const intervalMs = secondRunAt.getTime() - nextRunAt.getTime();+ if (intervalMs < MIN_MONITOR_INTERVAL_MS) {+ throw new Error(+ "Monitor schedule must not run more often than every 15 minutes",+ );+ }++ return { nextRunAt, intervalMs };+}++export function estimateRunsPerMonth(intervalMs: number): number {+ const daysPerMonth = 30;+ return Math.ceil((daysPerMonth * 24 * 60 * 60 * 1000) / intervalMs);+}diff --git a/apps/api/src/services/monitoring/diff.ts b/apps/api/src/services/monitoring/diff.tsnew file mode 100644index 000000000..3197367d3--- /dev/null+++ b/apps/api/src/services/monitoring/diff.ts@@ -0,0 +1,105 @@+import gitDiff from "git-diff";+import parseDiff from "parse-diff";++type MonitoringDiffResult =+ | {+ status: "same";+ text?: undefined;+ json?: undefined;+ }+ | {+ status: "changed";+ text: string;+ json: {+ files: Array<{+ from: string | null;+ to: string | null;+ chunks: Array<{+ content: string;+ changes: Array<{+ type: string;+ normal?: boolean;+ add?: boolean;+ del?: boolean;+ ln?: number;+ ln1?: number;+ ln2?: number;+ content: string;+ }>;+ }>;+ }>;+ };+ };++function normalizeMarkdownForChangeTracking(markdown: string): string {+ return [...markdown.replace(/\s+/g, "").replace(/\[iframe\]\(.+?\)/g, "")]+ .sort()+ .join("");+}++export function diffMonitorMarkdown(+ previousMarkdown: string,+ currentMarkdown: string,+): MonitoringDiffResult {+ if (+ normalizeMarkdownForChangeTracking(previousMarkdown) ===+ normalizeMarkdownForChangeTracking(currentMarkdown)+ ) {+ return { status: "same" };+ }++ const text = gitDiff(previousMarkdown, currentMarkdown, {+ color: false,+ wordDiff: false,+ });+ const structured = parseDiff(text);++ return {+ status: "changed",+ text,+ json: {+ files: structured.map(file => ({+ from: file.from || null,+ to: file.to || null,+ chunks: file.chunks.map(chunk => ({+ content: chunk.content,+ changes: chunk.changes.map(change => {+ const base = {+ type: change.type,+ content: change.content,+ };++ if (+ change.type === "normal" &&+ "ln1" in change &&+ "ln2" in change+ ) {+ return {+ ...base,+ normal: true,+ ln1: change.ln1,+ ln2: change.ln2,+ };+ }+ if (change.type === "add" && "ln" in change) {+ return {+ ...base,+ add: true,+ ln: change.ln,+ };+ }+ if (change.type === "del" && "ln" in change) {+ return {+ ...base,+ del: true,+ ln: change.ln,+ };+ }++ return base;+ }),+ })),+ })),+ },+ };+}diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsnew file mode 100644index 000000000..77cc8675f--- /dev/null+++ b/apps/api/src/services/monitoring/queue.ts@@ -0,0 +1,133 @@+import amqp from "amqplib";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";++const MONITOR_CHECK_QUEUE = "monitor.checks";+const MONITOR_CHECK_DLX = "monitor.checks.dlx";+const MONITOR_CHECK_DLQ = "monitor.checks.dlq";++const logger = _logger.child({ module: "monitoring-queue" });++export type MonitorCheckJobData = {+ monitorId: string;+ checkId: string;+ teamId: string;+};++let connection: amqp.ChannelModel | null = null;+let channel: amqp.Channel | null = null;++async function getChannel(): Promise<amqp.Channel> {+ if (channel) return channel;++ const url = config.NUQ_RABBITMQ_URL;+ if (!url) {+ throw new Error("NUQ_RABBITMQ_URL is not configured");+ }++ connection = await amqp.connect(url);+ channel = await connection.createChannel();++ await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });+ await channel.assertQueue(MONITOR_CHECK_DLQ, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ },+ });+ await channel.bindQueue(+ MONITOR_CHECK_DLQ,+ MONITOR_CHECK_DLX,+ MONITOR_CHECK_QUEUE,+ );++ await channel.assertQueue(MONITOR_CHECK_QUEUE, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-dead-letter-exchange": MONITOR_CHECK_DLX,+ "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,+ "x-delivery-limit": 1,+ },+ });++ connection.on("close", () => {+ logger.warn("Monitor queue connection closed");+ connection = null;+ channel = null;+ });++ connection.on("error", error => {+ logger.error("Monitor queue connection error", { error });+ });++ return channel;+}++export async function addMonitorCheckJob(+ data: MonitorCheckJobData,+): Promise<void> {+ const ch = await getChannel();+ const sent = ch.sendToQueue(+ MONITOR_CHECK_QUEUE,+ Buffer.from(JSON.stringify(data)),+ {+ persistent: true,+ contentType: "application/json",+ messageId: data.checkId,+ },+ );++ if (!sent) {+ logger.warn("Monitor check message buffer full", {+ monitorId: data.monitorId,+ checkId: data.checkId,+ });+ }++ logger.info("Monitor check job added to queue", {+ monitorId: data.monitorId,+ checkId: data.checkId,+ teamId: data.teamId,+ });+}++export async function consumeMonitorCheckJobs(+ handler: (data: MonitorCheckJobData) => Promise<void>,+): Promise<void> {+ const ch = await getChannel();+ await ch.prefetch(1);++ await ch.consume(+ MONITOR_CHECK_QUEUE,+ async msg => {+ if (!msg) return;++ let data: MonitorCheckJobData;+ try {+ data = JSON.parse(msg.content.toString()) as MonitorCheckJobData;+ } catch (error) {+ logger.error("Failed to parse monitor check job", { error });+ ch.nack(msg, false, false);+ return;+ }++ const jobLogger = logger.child({+ monitorId: data.monitorId,+ checkId: data.checkId,+ teamId: data.teamId,+ });++ try {+ await handler(data);+ ch.ack(msg);+ } catch (error) {+ jobLogger.error("Monitor check job failed", { error });+ ch.nack(msg, false, false);+ }+ },+ { noAck: false },+ );++ logger.info("Started consuming monitor check jobs");+}diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsnew file mode 100644index 000000000..2c06706f1--- /dev/null+++ b/apps/api/src/services/monitoring/results.ts@@ -0,0 +1,227 @@+import { v7 as uuidv7 } from "uuid";+import { NuQJob } from "../worker/nuq";+import { ScrapeJobData } from "../../types";+import { getJobFromGCS } from "../../lib/gcs-jobs";+import {+ monitorDiffGcsKey,+ saveMonitorDiffArtifact,+} from "../../lib/gcs-monitoring";+import { logger as _logger } from "../../lib/logger";+import { createWebhookSender, WebhookEvent } from "../webhook";+import { diffMonitorMarkdown } from "./diff";+import {+ getMonitorForUpdate,+ getMonitorPage,+ hashMonitorUrl,+ insertMonitorCheckPages,+ upsertMonitorPage,+} from "./store";++const logger = _logger.child({ module: "monitoring-results" });++function getDocumentUrl(doc: any, fallback: string): string {+ return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;+}++function getDocumentStatusCode(doc: any): number | null {+ return typeof doc?.metadata?.statusCode === "number"+ ? doc.metadata.statusCode+ : null;+}++async function sendMonitorPageWebhook(params: {+ teamId: string;+ monitorId: string;+ checkId: string;+ url: string;+ status: string;+ previousScrapeId?: string | null;+ currentScrapeId?: string | null;+ error?: string | null;+}) {+ try {+ const monitor = await getMonitorForUpdate(params.teamId, params.monitorId);+ if (!monitor?.webhook) return;++ const sender = await createWebhookSender({+ teamId: params.teamId,+ jobId: params.checkId,+ webhook: monitor.webhook as any,+ v0: false,+ });++ await sender?.send(WebhookEvent.MONITOR_PAGE, {+ success: params.status !== "error",+ data: {+ monitorId: params.monitorId,+ checkId: params.checkId,+ url: params.url,+ status: params.status,+ previousScrapeId: params.previousScrapeId ?? null,+ currentScrapeId: params.currentScrapeId ?? null,+ error: params.error ?? null,+ },+ error: params.error ?? undefined,+ });+ } catch (error) {+ logger.warn("Failed to send monitor page webhook", {+ error,+ monitorId: params.monitorId,+ checkId: params.checkId,+ url: params.url,+ status: params.status,+ });+ }+}++export async function recordMonitorScrapeSuccess(+ job: NuQJob<ScrapeJobData>,+ doc: any,+): Promise<void> {+ const monitoring = job.data.monitoring;+ if (!monitoring || job.data.mode !== "single_urls") return;++ const url = getDocumentUrl(doc, job.data.url);+ const previous = await getMonitorPage({+ monitorId: monitoring.monitorId,+ targetId: monitoring.targetId,+ url,+ });++ let status: "same" | "new" | "changed" = "new";+ let diffGcsKey: string | null = null;+ let diffTextBytes: number | null = null;+ let diffJsonBytes: number | null = null;++ if (previous?.last_scrape_id && !previous.is_removed) {+ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];+ const previousMarkdown = previousDoc?.markdown;+ const currentMarkdown = doc?.markdown;++ if (previousMarkdown && currentMarkdown) {+ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);+ status = diff.status;++ if (diff.status === "changed") {+ diffGcsKey = monitorDiffGcsKey({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ pageId: uuidv7(),+ });+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, {+ url,+ previousScrapeId: previous.last_scrape_id,+ currentScrapeId: job.id,+ text: diff.text,+ json: diff.json,+ generatedAt: new Date().toISOString(),+ });+ diffTextBytes = sizes.textBytes;+ diffJsonBytes = sizes.jsonBytes;+ }+ } else {+ status = "changed";+ }+ }++ await upsertMonitorPage({+ monitorId: monitoring.monitorId,+ teamId: job.data.team_id,+ targetId: monitoring.targetId,+ url,+ source: monitoring.source,+ checkId: monitoring.checkId,+ scrapeId: job.id,+ status,+ metadata: {+ title: doc?.metadata?.title ?? null,+ statusCode: getDocumentStatusCode(doc),+ creditsUsed: doc?.metadata?.creditsUsed ?? null,+ },+ });++ await insertMonitorCheckPages([+ {+ check_id: monitoring.checkId,+ monitor_id: monitoring.monitorId,+ team_id: job.data.team_id,+ target_id: monitoring.targetId,+ url,+ url_hash: hashMonitorUrl(url),+ status,+ previous_scrape_id: previous?.last_scrape_id ?? null,+ current_scrape_id: job.id,+ diff_gcs_key: diffGcsKey,+ diff_text_bytes: diffTextBytes,+ diff_json_bytes: diffJsonBytes,+ status_code: getDocumentStatusCode(doc),+ metadata: {+ title: doc?.metadata?.title ?? null,+ creditsUsed: doc?.metadata?.creditsUsed ?? null,+ },+ },+ ]);++ logger.info("Recorded monitor scrape result", {+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ scrapeId: job.id,+ url,+ status,+ previousScrapeId: previous?.last_scrape_id ?? null,+ diffGcsKey,+ });++ await sendMonitorPageWebhook({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ url,+ status,+ previousScrapeId: previous?.last_scrape_id ?? null,+ currentScrapeId: job.id,+ });+}++export async function recordMonitorScrapeFailure(+ job: NuQJob<ScrapeJobData>,+ error: unknown,+): Promise<void> {+ const monitoring = job.data.monitoring;+ if (!monitoring || job.data.mode !== "single_urls") return;++ await insertMonitorCheckPages([+ {+ check_id: monitoring.checkId,+ monitor_id: monitoring.monitorId,+ team_id: job.data.team_id,+ target_id: monitoring.targetId,+ url: job.data.url,+ url_hash: hashMonitorUrl(job.data.url),+ status: "error",+ current_scrape_id: job.id,+ error: error instanceof Error ? error.message : String(error),+ },+ ]);++ logger.info("Recorded monitor scrape failure", {+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ targetId: monitoring.targetId,+ scrapeId: job.id,+ url: job.data.url,+ error: error instanceof Error ? error.message : String(error),+ });++ await sendMonitorPageWebhook({+ teamId: job.data.team_id,+ monitorId: monitoring.monitorId,+ checkId: monitoring.checkId,+ url: job.data.url,+ status: "error",+ currentScrapeId: job.id,+ error: error instanceof Error ? error.message : String(error),+ });+}diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsnew file mode 100644index 000000000..4afa1ce73--- /dev/null+++ b/apps/api/src/services/monitoring/runner.ts@@ -0,0 +1,1098 @@+import { v7 as uuidv7 } from "uuid";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";+import { logRequest } from "../logging/log_job";+import { processJobInternal } from "../worker/scrape-worker";+import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";+import { ScrapeJobData } from "../../types";+import { getJobFromGCS } from "../../lib/gcs-jobs";+import {+ monitorDiffGcsKey,+ saveMonitorDiffArtifact,+} from "../../lib/gcs-monitoring";+import { diffMonitorMarkdown } from "./diff";+import { autumnService } from "../autumn/autumn.service";+import { getBillingQueue } from "../queue-service";+import {+ crawlToCrawler,+ markCrawlActive,+ saveCrawl,+ StoredCrawl,+} from "../../lib/crawl-redis";+import { _addScrapeJobToBullMQ, addScrapeJob } from "../queue-jobs";+import {+ CrawlRequest,+ type ScrapeOptions,+ crawlRequestSchema,+ scrapeRequestSchema,+ toV0CrawlerOptions,+} from "../../controllers/v2/types";+import { createWebhookSender, WebhookEvent } from "../webhook";+import { sendMonitoringEmailSummary } from "../notification/monitoring_email";+import {+ getMonitorForUpdate,+ getMonitorPage,+ countMonitorCheckPages,+ hashMonitorUrl,+ insertMonitorCheckPages,+ listActiveMonitorPages,+ listMonitorCheckPages,+ listRunningMonitorChecks,+ markMonitorRunning,+ updateMonitorCheck,+ updateMonitorScheduleAfterRun,+ upsertMonitorPage,+} from "./store";+import type {+ MonitorCheckPageInsert,+ MonitorCheckRow,+ MonitorRow,+ MonitorTarget,+} from "./types";+import { withMarkdownFormat } from "./types";+import { redisEvictConnection } from "../redis";+import type { MonitorCheckJobData } from "./queue";++const logger = _logger.child({ module: "monitoring-runner" });+const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));++type PageResult = MonitorCheckPageInsert & {+ emailStatus?: string;+};++function withMonitorScrapeDefaults(+ options: Record<string, unknown>,+): ScrapeOptions {+ return {+ maxAge: 0,+ ...withMarkdownFormat(options),+ };+}++function getDocumentUrl(doc: any, fallback: string): string {+ return doc?.metadata?.sourceURL ?? doc?.metadata?.url ?? doc?.url ?? fallback;+}++function getDocumentStatusCode(doc: any): number | null {+ return typeof doc?.metadata?.statusCode === "number"+ ? doc.metadata.statusCode+ : null;+}++function estimateActualCredits(doc: any, options: any): number {+ if (typeof doc?.metadata?.creditsUsed === "number") {+ return doc.metadata.creditsUsed;+ }+ const formats = Array.isArray(options?.formats) ? options.formats : [];+ const hasJson = formats.some((format: any) =>+ typeof format === "string" ? format === "json" : format?.type === "json",+ );+ return hasJson ? 5 : 1;+}++async function runSingleScrape(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ target: MonitorTarget;+ url: string;+ requestId?: string;+}): Promise<{ scrapeId: string; doc: any; credits: number }> {+ const scrapeId = uuidv7();+ const scrapeOptions = scrapeRequestSchema.parse({+ url: params.url,+ ...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),+ origin: "monitor",+ });++ await logRequest({+ id: scrapeId,+ kind: "scrape",+ api_version: "v2",+ team_id: params.monitor.team_id,+ origin: "monitor",+ integration: null,+ target_hint: params.url,+ zeroDataRetention: false,+ api_key_id: null,+ });++ const job: NuQJob<ScrapeJobData> = {+ id: scrapeId,+ status: "active",+ createdAt: new Date(),+ priority: 20,+ data: {+ mode: "single_urls",+ url: params.url,+ team_id: params.monitor.team_id,+ scrapeOptions,+ internalOptions: {+ teamId: params.monitor.team_id,+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,+ bypassBilling: true,+ zeroDataRetention: false,+ },+ skipNuq: true,+ origin: "monitor",+ integration: null,+ billing: { endpoint: "monitor", jobId: params.check.id },+ requestId: params.requestId,+ zeroDataRetention: false,+ apiKeyId: null,+ },+ };++ const doc = await processJobInternal(job);+ return {+ scrapeId,+ doc,+ credits: estimateActualCredits(doc, scrapeOptions),+ };+}++async function diffAndPersistPage(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ target: MonitorTarget;+ url: string;+ scrapeId: string;+ doc: any;+ source: "explicit" | "discovered";+}): Promise<PageResult> {+ const previous = await getMonitorPage({+ monitorId: params.monitor.id,+ targetId: params.target.id,+ url: params.url,+ });++ let status: PageResult["status"] = "new";+ let diffGcsKey: string | null = null;+ let diffTextBytes: number | null = null;+ let diffJsonBytes: number | null = null;++ if (previous?.last_scrape_id && !previous.is_removed) {+ const previousDoc = (await getJobFromGCS(previous.last_scrape_id))?.[0];+ const previousMarkdown = previousDoc?.markdown;+ const currentMarkdown = params.doc?.markdown;++ if (previousMarkdown && currentMarkdown) {+ const diff = diffMonitorMarkdown(previousMarkdown, currentMarkdown);+ status = diff.status;++ if (diff.status === "changed") {+ diffGcsKey = monitorDiffGcsKey({+ teamId: params.monitor.team_id,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ pageId: uuidv7(),+ });+ const sizes = await saveMonitorDiffArtifact(diffGcsKey, {+ url: params.url,+ previousScrapeId: previous.last_scrape_id,+ currentScrapeId: params.scrapeId,+ text: diff.text,+ json: diff.json,+ generatedAt: new Date().toISOString(),+ });+ diffTextBytes = sizes.textBytes;+ diffJsonBytes = sizes.jsonBytes;+ }+ } else {+ status = "changed";+ }+ }++ await upsertMonitorPage({+ monitorId: params.monitor.id,+ teamId: params.monitor.team_id,+ targetId: params.target.id,+ url: params.url,+ source: params.source,+ checkId: params.check.id,+ scrapeId: params.scrapeId,+ status,+ metadata: {+ title: params.doc?.metadata?.title ?? null,+ statusCode: getDocumentStatusCode(params.doc),+ },+ });++ return {+ check_id: params.check.id,+ monitor_id: params.monitor.id,+ team_id: params.monitor.team_id,+ target_id: params.target.id,+ url: params.url,+ url_hash: hashMonitorUrl(params.url),+ status,+ previous_scrape_id: previous?.last_scrape_id ?? null,+ current_scrape_id: params.scrapeId,+ diff_gcs_key: diffGcsKey,+ diff_text_bytes: diffTextBytes,+ diff_json_bytes: diffJsonBytes,+ status_code: getDocumentStatusCode(params.doc),+ metadata: {+ title: params.doc?.metadata?.title ?? null,+ },+ emailStatus: status,+ };+}++async function runScrapeTarget(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ target: MonitorTarget;+}): Promise<{ pages: PageResult[]; credits: number; targetResult: any }> {+ if (params.target.type !== "scrape") {+ return { pages: [], credits: 0, targetResult: null };+ }++ const pages: PageResult[] = [];+ let credits = 0;++ for (const url of params.target.urls) {+ try {+ const result = await runSingleScrape({+ monitor: params.monitor,+ check: params.check,+ target: params.target,+ url,+ });+ credits += result.credits;+ pages.push(+ await diffAndPersistPage({+ monitor: params.monitor,+ check: params.check,+ target: params.target,+ url,+ scrapeId: result.scrapeId,+ doc: result.doc,+ source: "explicit",+ }),+ );+ } catch (error) {+ pages.push({+ check_id: params.check.id,+ monitor_id: params.monitor.id,+ team_id: params.monitor.team_id,+ target_id: params.target.id,+ url,+ url_hash: hashMonitorUrl(url),+ status: "error",+ error: error instanceof Error ? error.message : String(error),+ emailStatus: "error",+ });+ }+ }++ return {+ pages,+ credits,+ targetResult: {+ targetId: params.target.id,+ type: params.target.type,+ pages: pages.length,+ credits,+ },+ };+}++async function runCrawlTarget(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ target: MonitorTarget;+}): Promise<{ pages: PageResult[]; credits: number; targetResult: any }> {+ if (params.target.type !== "crawl") {+ return { pages: [], credits: 0, targetResult: null };+ }++ const crawlId = uuidv7();+ const body = crawlRequestSchema.parse({+ url: params.target.url,+ ...(params.target.crawlOptions ?? {}),+ scrapeOptions: withMarkdownFormat(params.target.scrapeOptions ?? {}),+ origin: "monitor",+ }) as CrawlRequest;++ await logRequest({+ id: crawlId,+ kind: "crawl",+ api_version: "v2",+ team_id: params.monitor.team_id,+ origin: "monitor",+ integration: null,+ target_hint: body.url,+ zeroDataRetention: false,+ api_key_id: null,+ });++ const crawlerOptions = {+ ...body,+ url: undefined,+ scrapeOptions: undefined,+ prompt: undefined,+ };++ const sc: StoredCrawl = {+ originUrl: body.url,+ crawlerOptions: toV0CrawlerOptions(crawlerOptions),+ scrapeOptions: body.scrapeOptions,+ internalOptions: {+ disableSmartWaitCache: true,+ teamId: params.monitor.team_id,+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,+ zeroDataRetention: false,+ bypassBilling: true,+ },+ team_id: params.monitor.team_id,+ createdAt: Date.now(),+ maxConcurrency: body.maxConcurrency,+ zeroDataRetention: false,+ };++ const crawler = crawlToCrawler(crawlId, sc, null);+ try {+ sc.robots = await crawler.getRobotsTxt(+ body.scrapeOptions.skipTlsVerification,+ );+ } catch {+ // Crawls tolerate robots fetch failures in the public controller too.+ }++ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);+ await saveCrawl(crawlId, sc);+ await markCrawlActive(crawlId);++ await _addScrapeJobToBullMQ(+ {+ url: body.url,+ mode: "kickoff",+ team_id: params.monitor.team_id,+ crawlerOptions,+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: "monitor",+ integration: null,+ billing: { endpoint: "monitor", jobId: params.check.id },+ crawl_id: crawlId,+ v1: true,+ zeroDataRetention: false,+ apiKeyId: null,+ },+ uuidv7(),+ );++ const started = Date.now();+ let status = "scraping";+ let total = 0;+ while (Date.now() - started < 30 * 60 * 1000) {+ const group = await crawlGroup.getGroup(crawlId);+ const stats = await scrapeQueue.getGroupNumericStats(crawlId, logger);+ status = group?.status ?? "scraping";+ total =+ (stats.completed ?? 0) ++ (stats.active ?? 0) ++ (stats.queued ?? 0) ++ (stats.backlog ?? 0);+ if (status !== "active" && status !== "scraping") break;+ await poll(1000);+ }++ const doneJobs = await scrapeQueue.getCrawlJobsForListing(+ crawlId,+ Math.max(total, 1),+ 0,+ logger,+ );++ const pages: PageResult[] = [];+ const seen = new Set<string>();+ let credits = 0;++ for (const job of doneJobs) {+ const doc = job.returnvalue ?? (await getJobFromGCS(job.id))?.[0];+ if (!doc) continue;+ const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);+ seen.add(hashMonitorUrl(url));+ credits += estimateActualCredits(doc, body.scrapeOptions);+ pages.push(+ await diffAndPersistPage({+ monitor: params.monitor,+ check: params.check,+ target: params.target,+ url,+ scrapeId: job.id,+ doc,+ source: "discovered",+ }),+ );+ }++ if (status === "completed") {+ const previousPages = await listActiveMonitorPages({+ monitorId: params.monitor.id,+ targetId: params.target.id,+ });+ for (const previous of previousPages) {+ if (seen.has(previous.url_hash)) continue;+ await upsertMonitorPage({+ monitorId: params.monitor.id,+ teamId: params.monitor.team_id,+ targetId: params.target.id,+ url: previous.url,+ source: previous.source,+ checkId: params.check.id,+ scrapeId: previous.last_scrape_id,+ status: "removed",+ metadata: previous.metadata,+ });+ pages.push({+ check_id: params.check.id,+ monitor_id: params.monitor.id,+ team_id: params.monitor.team_id,+ target_id: params.target.id,+ url: previous.url,+ url_hash: previous.url_hash,+ status: "removed",+ previous_scrape_id: previous.last_scrape_id,+ current_scrape_id: null,+ emailStatus: "removed",+ });+ }+ }++ return {+ pages,+ credits,+ targetResult: {+ targetId: params.target.id,+ type: params.target.type,+ crawlId,+ status,+ pages: pages.length,+ credits,+ },+ };+}++function summarize(pages: PageResult[]) {+ return {+ totalPages: pages.length,+ same: pages.filter(page => page.status === "same").length,+ changed: pages.filter(page => page.status === "changed").length,+ new: pages.filter(page => page.status === "new").length,+ removed: pages.filter(page => page.status === "removed").length,+ error: pages.filter(page => page.status === "error").length,+ };+}++async function billMonitorCheck(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ actualCredits: number;+ lockId: string | null;+}): Promise<void> {+ if (params.lockId) {+ await autumnService.finalizeCreditsLock({+ lockId: params.lockId,+ action: "confirm",+ overrideValue: params.actualCredits,+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: params.check.id,+ },+ });+ }++ if (params.actualCredits <= 0 || !config.USE_DB_AUTHENTICATION) return;++ await getBillingQueue().add(+ "bill_team",+ {+ team_id: params.monitor.team_id,+ subscription_id: undefined,+ credits: params.actualCredits,+ billing: { endpoint: "monitor", jobId: params.check.id },+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: params.check.id,+ api_key_id: null,+ autumnTrackInRequest: Boolean(params.lockId),+ },+ {+ jobId: uuidv7(),+ priority: 10,+ },+ );+}++async function sendNotifications(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ pages: PageResult[];+}): Promise<{ webhook?: unknown; email?: unknown }> {+ const payload = {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ status: params.check.status,+ summary: toSummaryObject(params.check),+ };++ let webhookStatus: unknown = { attempted: false };+ if (params.monitor.webhook) {+ const sender = await createWebhookSender({+ teamId: params.monitor.team_id,+ jobId: params.check.id,+ webhook: params.monitor.webhook as any,+ v0: false,+ });+ await sender?.send(WebhookEvent.MONITOR_CHECK_COMPLETED, {+ success: params.check.status === "completed",+ data: payload,+ error: params.check.error ?? undefined,+ });+ webhookStatus = { attempted: true, success: true };+ }++ const emailStatus = await sendMonitoringEmailSummary({+ monitor: params.monitor,+ check: params.check,+ pages: params.pages+ .filter(page => page.status !== "same")+ .slice(0, 25)+ .map(page => ({+ url: page.url,+ status: page.status,+ error: page.error,+ })),+ });++ return {+ webhook: webhookStatus,+ email: emailStatus,+ };+}++async function enqueueMonitorScrapeTarget(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ target: MonitorTarget;+}): Promise<{ targetId: string; type: "scrape"; expectedJobs: string[] }> {+ if (params.target.type !== "scrape") {+ throw new Error("Expected scrape target");+ }++ const expectedJobs: string[] = [];+ for (const url of params.target.urls) {+ const scrapeId = uuidv7();+ const scrapeOptions = scrapeRequestSchema.parse({+ url,+ ...withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),+ origin: "monitor",+ });++ await logRequest({+ id: scrapeId,+ kind: "scrape",+ api_version: "v2",+ team_id: params.monitor.team_id,+ origin: "monitor",+ integration: null,+ target_hint: url,+ zeroDataRetention: false,+ api_key_id: null,+ });++ await addScrapeJob(+ {+ mode: "single_urls",+ url,+ team_id: params.monitor.team_id,+ scrapeOptions,+ internalOptions: {+ teamId: params.monitor.team_id,+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,+ bypassBilling: true,+ zeroDataRetention: false,+ },+ origin: "monitor",+ integration: null,+ billing: { endpoint: "monitor", jobId: params.check.id },+ zeroDataRetention: false,+ apiKeyId: null,+ monitoring: {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ targetId: params.target.id,+ source: "explicit",+ },+ },+ scrapeId,+ 20,+ );+ expectedJobs.push(scrapeId);+ }++ return { targetId: params.target.id, type: "scrape", expectedJobs };+}++async function enqueueMonitorCrawlTarget(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ target: MonitorTarget;+}): Promise<{ targetId: string; type: "crawl"; crawlId: string }> {+ if (params.target.type !== "crawl") {+ throw new Error("Expected crawl target");+ }++ const crawlId = uuidv7();+ const body = crawlRequestSchema.parse({+ url: params.target.url,+ ...(params.target.crawlOptions ?? {}),+ scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),+ origin: "monitor",+ }) as CrawlRequest;++ await logRequest({+ id: crawlId,+ kind: "crawl",+ api_version: "v2",+ team_id: params.monitor.team_id,+ origin: "monitor",+ integration: null,+ target_hint: body.url,+ zeroDataRetention: false,+ api_key_id: null,+ });++ const crawlerOptions = {+ ...body,+ url: undefined,+ scrapeOptions: undefined,+ prompt: undefined,+ };++ const sc: StoredCrawl = {+ originUrl: body.url,+ crawlerOptions: toV0CrawlerOptions(crawlerOptions),+ scrapeOptions: body.scrapeOptions,+ internalOptions: {+ disableSmartWaitCache: true,+ teamId: params.monitor.team_id,+ saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,+ zeroDataRetention: false,+ bypassBilling: true,+ },+ team_id: params.monitor.team_id,+ createdAt: Date.now(),+ maxConcurrency: body.maxConcurrency,+ zeroDataRetention: false,+ };++ const crawler = crawlToCrawler(crawlId, sc, null);+ try {+ sc.robots = await crawler.getRobotsTxt(+ body.scrapeOptions.skipTlsVerification,+ );+ } catch {+ // Non-fatal, same as the public crawl controller.+ }++ await crawlGroup.addGroup(crawlId, sc.team_id, 24 * 60 * 60 * 1000);+ await saveCrawl(crawlId, sc);+ await markCrawlActive(crawlId);++ await _addScrapeJobToBullMQ(+ {+ url: body.url,+ mode: "kickoff",+ team_id: params.monitor.team_id,+ crawlerOptions,+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: "monitor",+ integration: null,+ billing: { endpoint: "monitor", jobId: params.check.id },+ crawl_id: crawlId,+ v1: true,+ zeroDataRetention: false,+ apiKeyId: null,+ monitoring: {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ targetId: params.target.id,+ source: "discovered",+ },+ },+ uuidv7(),+ );++ return { targetId: params.target.id, type: "crawl", crawlId };+}++export async function processMonitorCheckJob(+ job: MonitorCheckJobData,+): Promise<void> {+ const monitor = await getMonitorForUpdate(job.teamId, job.monitorId);+ if (!monitor) {+ throw new Error("Monitor not found");+ }++ await markMonitorRunning({+ monitorId: monitor.id,+ checkId: job.checkId,+ });++ let check = await updateMonitorCheck(job.checkId, {+ status: "running",+ started_at: new Date().toISOString(),+ });++ let lockId: string | null = null;+ try {+ lockId = await autumnService.lockCredits({+ teamId: monitor.team_id,+ value: check.estimated_credits ?? 1,+ lockId: `monitor_${check.id}`,+ expiresAt: Date.now() + 60 * 60 * 1000,+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: check.id,+ },+ });++ check = await updateMonitorCheck(check.id, {+ autumn_lock_id: lockId,+ reserved_credits: lockId ? (check.estimated_credits ?? 1) : null,+ billing_status: lockId ? "reserved" : "not_applicable",+ });++ const targetResults: unknown[] = [];++ for (const target of monitor.targets) {+ const result =+ target.type === "scrape"+ ? await enqueueMonitorScrapeTarget({ monitor, check, target })+ : await enqueueMonitorCrawlTarget({ monitor, check, target });+ targetResults.push(result);+ }++ await updateMonitorCheck(check.id, {+ target_results: targetResults,+ });+ } catch (error) {+ if (lockId) {+ await autumnService.finalizeCreditsLock({+ lockId,+ action: "release",+ properties: {+ source: "monitorCheck",+ endpoint: "monitor",+ jobId: check.id,+ },+ });+ }++ check = await updateMonitorCheck(check.id, {+ status: "failed",+ finished_at: new Date().toISOString(),+ billing_status: lockId ? "released" : "failed",+ error: error instanceof Error ? error.message : String(error),+ });++ await sendNotifications({+ monitor,+ check,+ pages: [],+ }).catch(err =>+ logger.warn("Failed to send monitor failure notifications", {+ error: err,+ }),+ );++ await updateMonitorScheduleAfterRun({+ monitor,+ check,+ });++ throw error;+ }+}++async function processRemovedPagesForCompletedCrawls(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ targetResults: any[];+}): Promise<void> {+ for (const target of params.targetResults) {+ if (target?.type !== "crawl" || target.removedProcessed) continue;++ const group = await crawlGroup.getGroup(target.crawlId);+ if (group?.status !== "completed") continue;++ const checkPages = await listMonitorCheckPages({+ teamId: params.monitor.team_id,+ monitorId: params.monitor.id,+ checkId: params.check.id,+ limit: 100000,+ skip: 0,+ });+ const seen = new Set(+ checkPages+ .filter(page => page.target_id === target.targetId)+ .map(page => page.url_hash),+ );+ const activePages = await listActiveMonitorPages({+ monitorId: params.monitor.id,+ targetId: target.targetId,+ });++ const removed: MonitorCheckPageInsert[] = [];+ for (const previous of activePages) {+ if (seen.has(previous.url_hash)) continue;+ await upsertMonitorPage({+ monitorId: params.monitor.id,+ teamId: params.monitor.team_id,+ targetId: target.targetId,+ url: previous.url,+ source: previous.source,+ checkId: params.check.id,+ scrapeId: previous.last_scrape_id,+ status: "removed",+ metadata: previous.metadata,+ });+ removed.push({+ check_id: params.check.id,+ monitor_id: params.monitor.id,+ team_id: params.monitor.team_id,+ target_id: target.targetId,+ url: previous.url,+ url_hash: previous.url_hash,+ status: "removed" as const,+ previous_scrape_id: previous.last_scrape_id,+ current_scrape_id: null,+ });+ }++ await insertMonitorCheckPages(removed);+ target.removedProcessed = true;+ }+}++async function isMonitorCheckComplete(+ check: MonitorCheckRow,+): Promise<boolean> {+ const targetResults = Array.isArray(check.target_results)+ ? (check.target_results as any[])+ : [];++ if (targetResults.length === 0) return false;++ for (const target of targetResults) {+ if (target?.type === "scrape") {+ const expected = Array.isArray(target.expectedJobs)+ ? target.expectedJobs.length+ : 0;+ const recorded = await countMonitorCheckPages({+ checkId: check.id,+ targetId: target.targetId,+ });+ if (recorded < expected) return false;+ } else if (target?.type === "crawl") {+ const group = await crawlGroup.getGroup(target.crawlId);+ if (!group || group.status === "active") return false;++ const stats = await scrapeQueue.getGroupNumericStats(+ target.crawlId,+ logger,+ );+ const unfinished =+ (stats.active ?? 0) + (stats.queued ?? 0) + (stats.backlog ?? 0);+ if (unfinished > 0) return false;+ }+ }++ return true;+}++export async function reconcileRunningMonitorChecks(+ limit: number = 50,+): Promise<void> {+ const checks = await listRunningMonitorChecks(limit);+ for (const check of checks) {+ const lockKey = `monitor-check-finalize:${check.id}`;+ const lock = await redisEvictConnection.set(lockKey, "1", "EX", 60, "NX");+ if (lock !== "OK") continue;++ try {+ const monitor = await getMonitorForUpdate(+ check.team_id,+ check.monitor_id,+ );+ if (!monitor) continue;++ const targetResults = Array.isArray(check.target_results)+ ? ([...check.target_results] as any[])+ : [];++ await processRemovedPagesForCompletedCrawls({+ monitor,+ check,+ targetResults,+ });++ if (+ !(await isMonitorCheckComplete({+ ...check,+ target_results: targetResults,+ }))+ ) {+ await updateMonitorCheck(check.id, { target_results: targetResults });+ continue;+ }++ const [same, changed, newCount, removed, errorCount] = await Promise.all([+ countMonitorCheckPages({ checkId: check.id, status: "same" }),+ countMonitorCheckPages({ checkId: check.id, status: "changed" }),+ countMonitorCheckPages({ checkId: check.id, status: "new" }),+ countMonitorCheckPages({ checkId: check.id, status: "removed" }),+ countMonitorCheckPages({ checkId: check.id, status: "error" }),+ ]);+ const totalPages = same + changed + newCount + removed + errorCount;+ const actualCredits = totalPages;++ let finalized = await updateMonitorCheck(check.id, {+ status: errorCount > 0 ? "partial" : "completed",+ finished_at: new Date().toISOString(),+ actual_credits: actualCredits,+ billing_status: check.autumn_lock_id ? "confirmed" : "not_applicable",+ total_pages: totalPages,+ same_count: same,+ changed_count: changed,+ new_count: newCount,+ removed_count: removed,+ error_count: errorCount,+ target_results: targetResults,+ });++ try {+ await billMonitorCheck({+ monitor,+ check: finalized,+ actualCredits,+ lockId: check.autumn_lock_id,+ });+ } catch (error) {+ logger.warn("Failed to bill monitor check during reconciliation", {+ monitorId: monitor.id,+ checkId: finalized.id,+ error,+ });+ finalized = await updateMonitorCheck(check.id, {+ billing_status: "failed",+ }).catch(updateError => {+ logger.warn("Failed to record monitor check billing failure", {+ monitorId: monitor.id,+ checkId: finalized.id,+ error: updateError,+ });+ return finalized;+ });+ }++ let notificationStatus: { webhook?: unknown; email?: unknown } | null =+ null;+ try {+ const pages = (await listMonitorCheckPages({+ teamId: monitor.team_id,+ monitorId: monitor.id,+ checkId: check.id,+ limit: 100,+ skip: 0,+ })) as PageResult[];++ notificationStatus = await sendNotifications({+ monitor,+ check: finalized,+ pages,+ });++ finalized = await updateMonitorCheck(check.id, {+ notification_status: notificationStatus,+ webhook_payload: notificationStatus.webhook+ ? { summary: toSummaryObject(finalized) }+ : null,+ email_payload: notificationStatus.email+ ? { summary: toSummaryObject(finalized) }+ : null,+ });+ } catch (error) {+ logger.warn("Failed to send monitor check notifications", {+ monitorId: monitor.id,+ checkId: finalized.id,+ error,+ });+ notificationStatus = {+ webhook: {+ attempted: !!monitor.webhook,+ success: false,+ error: error instanceof Error ? error.message : String(error),+ },+ email: {+ attempted: !!monitor.notification?.email?.enabled,+ success: false,+ error: error instanceof Error ? error.message : String(error),+ },+ };+ finalized = await updateMonitorCheck(check.id, {+ notification_status: notificationStatus,+ }).catch(updateError => {+ logger.warn("Failed to record monitor check notification failure", {+ monitorId: monitor.id,+ checkId: finalized.id,+ error: updateError,+ });+ return finalized;+ });+ }++ await updateMonitorScheduleAfterRun({+ monitor,+ check: finalized,+ summary: toSummaryObject(finalized),+ });++ logger.info("Reconciled monitor check", {+ monitorId: monitor.id,+ checkId: finalized.id,+ status: finalized.status,+ totalPages,+ same,+ changed,+ new: newCount,+ removed,+ errors: errorCount,+ });+ } catch (error) {+ logger.warn("Failed to reconcile monitor check", {+ error,+ checkId: check.id,+ });+ } finally {+ await redisEvictConnection.del(lockKey);+ }+ }+}++function toSummaryObject(check: MonitorCheckRow) {+ return {+ totalPages: check.total_pages,+ same: check.same_count,+ changed: check.changed_count,+ new: check.new_count,+ removed: check.removed_count,+ error: check.error_count,+ };+}diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsnew file mode 100644index 000000000..99b817b34--- /dev/null+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -0,0 +1,75 @@+import { randomUUID } from "crypto";+import { logger as _logger } from "../../lib/logger";+import { addMonitorCheckJob } from "./queue";+import {+ advanceMonitorAfterSkippedCheck,+ claimDueMonitors,+ createMonitorCheck,+ updateMonitorCheck,+} from "./store";++const logger = _logger.child({ module: "monitoring-scheduler" });++export async function enqueueMonitorCheck(params: {+ monitorId: string;+ checkId: string;+ teamId: string;+}): Promise<void> {+ await addMonitorCheckJob(params);+}++export async function enqueueDueMonitorChecks(+ params: {+ workerId?: string;+ limit?: number;+ leaseSeconds?: number;+ } = {},+): Promise<number> {+ const workerId = params.workerId ?? `monitor-scheduler-${randomUUID()}`;+ const monitors = await claimDueMonitors({+ workerId,+ limit: params.limit ?? 10,+ leaseSeconds: params.leaseSeconds ?? 60,+ });++ let enqueued = 0;+ for (const monitor of monitors) {+ try {+ if (monitor.current_check_id) {+ const skipped = await createMonitorCheck({+ monitor,+ trigger: "scheduled",+ scheduledFor: monitor.next_run_at,+ status: "skipped_overlap",+ });+ const finished = await updateMonitorCheck(skipped.id, {+ status: "skipped_overlap",+ finished_at: new Date().toISOString(),+ error: "Previous monitor check is still running.",+ });+ await advanceMonitorAfterSkippedCheck({ monitor, check: finished });+ continue;+ }++ const check = await createMonitorCheck({+ monitor,+ trigger: "scheduled",+ scheduledFor: monitor.next_run_at,+ });+ await enqueueMonitorCheck({+ monitorId: monitor.id,+ checkId: check.id,+ teamId: monitor.team_id,+ });+ enqueued++;+ } catch (error) {+ logger.error("Failed to enqueue due monitor check", {+ error,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ }+ }++ return enqueued;+}diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsnew file mode 100644index 000000000..2d4e51dff--- /dev/null+++ b/apps/api/src/services/monitoring/store.ts@@ -0,0 +1,589 @@+import { createHash } from "crypto";+import { v7 as uuidv7 } from "uuid";+import { supabase_rr_service, supabase_service } from "../supabase";+import { getNextMonitorRunAt, estimateRunsPerMonth } from "./cron";+import type {+ CreateMonitorRequest,+ MonitorCheckPageInsert,+ MonitorCheckRow,+ MonitorPageRow,+ MonitorRow,+ MonitorSummary,+ MonitorTarget,+ UpdateMonitorRequest,+} from "./types";++export function hashMonitorUrl(url: string): string {+ return `\\x${createHash("sha256").update(url).digest("hex")}`;+}++function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {+ return targets.map(target => ({+ ...target,+ id: typeof target.id === "string" ? target.id : uuidv7(),+ })) as MonitorTarget[];+}++function estimateTargetCredits(target: MonitorTarget): number {+ if (target.type === "scrape") {+ return target.urls.length;+ }++ const limit =+ typeof target.crawlOptions?.limit === "number"+ ? target.crawlOptions.limit+ : 10000;+ return Math.max(1, limit);+}++export function estimateMonitorCreditsPerRun(targets: MonitorTarget[]): number {+ return targets.reduce(+ (sum, target) => sum + estimateTargetCredits(target),+ 0,+ );+}++function toMonitorSummary(check: MonitorCheckRow): MonitorSummary {+ return {+ totalPages: check.total_pages,+ same: check.same_count,+ changed: check.changed_count,+ new: check.new_count,+ removed: check.removed_count,+ error: check.error_count,+ };+}++function throwIfError(error: any, message: string): void {+ if (error) {+ throw new Error(`${message}: ${error.message ?? JSON.stringify(error)}`);+ }+}++export async function createMonitor(params: {+ teamId: string;+ input: CreateMonitorRequest;+ nextRunAt: Date;+ intervalMs: number;+}): Promise<MonitorRow> {+ const targets = ensureTargetIds(params.input.targets);+ const estimatedCreditsPerRun = estimateMonitorCreditsPerRun(targets);+ const estimatedCreditsPerMonth =+ estimatedCreditsPerRun * estimateRunsPerMonth(params.intervalMs);++ const { data, error } = await supabase_service+ .from("monitors")+ .insert({+ id: uuidv7(),+ team_id: params.teamId,+ name: params.input.name,+ schedule_cron: params.input.schedule.cron,+ schedule_timezone: params.input.schedule.timezone,+ next_run_at: params.nextRunAt.toISOString(),+ retention_days: params.input.retentionDays,+ estimated_credits_per_month: estimatedCreditsPerMonth,+ targets,+ webhook: params.input.webhook ?? null,+ notification: params.input.notification ?? null,+ })+ .select("*")+ .single();++ throwIfError(error, "Failed to create monitor");+ return data as MonitorRow;+}++export async function listMonitors(params: {+ teamId: string;+ limit: number;+ offset: number;+}): Promise<MonitorRow[]> {+ const { data, error } = await supabase_rr_service+ .from("monitors")+ .select("*")+ .eq("team_id", params.teamId)+ .neq("status", "deleted")+ .order("created_at", { ascending: false })+ .range(params.offset, params.offset + params.limit - 1);++ throwIfError(error, "Failed to list monitors");+ return (data ?? []) as MonitorRow[];+}++export async function getMonitor(+ teamId: string,+ monitorId: string,+): Promise<MonitorRow | null> {+ const { data, error } = await supabase_rr_service+ .from("monitors")+ .select("*")+ .eq("id", monitorId)+ .eq("team_id", teamId)+ .neq("status", "deleted")+ .maybeSingle();++ throwIfError(error, "Failed to get monitor");+ return data as MonitorRow | null;+}++export async function getMonitorForUpdate(+ teamId: string,+ monitorId: string,+): Promise<MonitorRow | null> {+ const { data, error } = await supabase_service+ .from("monitors")+ .select("*")+ .eq("id", monitorId)+ .eq("team_id", teamId)+ .neq("status", "deleted")+ .maybeSingle();++ throwIfError(error, "Failed to get monitor");+ return data as MonitorRow | null;+}++export async function updateMonitor(params: {+ teamId: string;+ monitorId: string;+ input: UpdateMonitorRequest;+ nextRunAt?: Date;+ intervalMs?: number;+}): Promise<MonitorRow | null> {+ const patch: Record<string, unknown> = {+ updated_at: new Date().toISOString(),+ };++ if (params.input.name !== undefined) patch.name = params.input.name;+ if (params.input.status !== undefined) patch.status = params.input.status;+ if (params.input.webhook !== undefined)+ patch.webhook = params.input.webhook ?? null;+ if (params.input.notification !== undefined) {+ patch.notification = params.input.notification ?? null;+ }+ if (params.input.retentionDays !== undefined) {+ patch.retention_days = params.input.retentionDays;+ }+ if (params.input.targets !== undefined) {+ const targets = ensureTargetIds(params.input.targets);+ patch.targets = targets;+ if (params.intervalMs !== undefined) {+ patch.estimated_credits_per_month =+ estimateMonitorCreditsPerRun(targets) *+ estimateRunsPerMonth(params.intervalMs);+ }+ }+ if (params.input.schedule !== undefined) {+ patch.schedule_cron = params.input.schedule.cron;+ patch.schedule_timezone = params.input.schedule.timezone;+ patch.next_run_at = params.nextRunAt?.toISOString() ?? null;+ }++ const { data, error } = await supabase_service+ .from("monitors")+ .update(patch)+ .eq("id", params.monitorId)+ .eq("team_id", params.teamId)+ .neq("status", "deleted")+ .select("*")+ .maybeSingle();++ throwIfError(error, "Failed to update monitor");+ return data as MonitorRow | null;+}++export async function deleteMonitor(params: {+ teamId: string;+ monitorId: string;+}): Promise<boolean> {+ const { data, error } = await supabase_service+ .from("monitors")+ .update({+ status: "deleted",+ deleted_at: new Date().toISOString(),+ next_run_at: null,+ updated_at: new Date().toISOString(),+ })+ .eq("id", params.monitorId)+ .eq("team_id", params.teamId)+ .neq("status", "deleted")+ .select("id")+ .maybeSingle();++ throwIfError(error, "Failed to delete monitor");+ return !!data;+}++export async function createMonitorCheck(params: {+ monitor: MonitorRow;+ trigger: "scheduled" | "manual";+ scheduledFor?: string | null;+ status?: MonitorCheckRow["status"];+}): Promise<MonitorCheckRow> {+ const estimated = estimateMonitorCreditsPerRun(params.monitor.targets);+ const { data, error } = await supabase_service+ .from("monitor_checks")+ .insert({+ id: uuidv7(),+ monitor_id: params.monitor.id,+ team_id: params.monitor.team_id,+ trigger: params.trigger,+ status: params.status ?? "queued",+ scheduled_for: params.scheduledFor ?? null,+ estimated_credits: estimated,+ })+ .select("*")+ .single();++ throwIfError(error, "Failed to create monitor check");+ return data as MonitorCheckRow;+}++export async function markMonitorRunning(params: {+ monitorId: string;+ checkId: string;+}): Promise<void> {+ const { error } = await supabase_service+ .from("monitors")+ .update({+ current_check_id: params.checkId,+ updated_at: new Date().toISOString(),+ })+ .eq("id", params.monitorId)+ .is("current_check_id", null);++ throwIfError(error, "Failed to mark monitor running");+}++export async function updateMonitorScheduleAfterRun(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ summary?: MonitorSummary;+}): Promise<void> {+ const nextRunAt =+ params.monitor.status === "active"+ ? getNextMonitorRunAt(+ params.monitor.schedule_cron,+ new Date(),+ params.monitor.schedule_timezone,+ ).toISOString()+ : null;+ const { error } = await supabase_service+ .from("monitors")+ .update({+ current_check_id: null,+ locked_at: null,+ locked_until: null,+ last_run_at: params.check.finished_at ?? new Date().toISOString(),+ last_check_id: params.check.id,+ next_run_at: nextRunAt,+ last_check_summary: params.summary ?? toMonitorSummary(params.check),+ updated_at: new Date().toISOString(),+ })+ .eq("id", params.monitor.id);++ throwIfError(error, "Failed to update monitor after run");+}++export async function advanceMonitorAfterSkippedCheck(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+}): Promise<void> {+ const nextRunAt =+ params.monitor.status === "active"+ ? getNextMonitorRunAt(+ params.monitor.schedule_cron,+ new Date(),+ params.monitor.schedule_timezone,+ ).toISOString()+ : null;+ const { error } = await supabase_service+ .from("monitors")+ .update({+ locked_at: null,+ locked_until: null,+ last_run_at: params.check.finished_at ?? new Date().toISOString(),+ last_check_id: params.check.id,+ next_run_at: nextRunAt,+ last_check_summary: toMonitorSummary(params.check),+ updated_at: new Date().toISOString(),+ })+ .eq("id", params.monitor.id);++ throwIfError(error, "Failed to advance monitor after skipped check");+}++export async function getMonitorCheck(+ teamId: string,+ monitorId: string,+ checkId: string,+): Promise<MonitorCheckRow | null> {+ const { data, error } = await supabase_rr_service+ .from("monitor_checks")+ .select("*")+ .eq("id", checkId)+ .eq("monitor_id", monitorId)+ .eq("team_id", teamId)+ .maybeSingle();++ throwIfError(error, "Failed to get monitor check");+ return data as MonitorCheckRow | null;+}++export async function listRunningMonitorChecks(+ limit: number = 100,+): Promise<MonitorCheckRow[]> {+ const { data, error } = await supabase_service+ .from("monitor_checks")+ .select("*")+ .eq("status", "running")+ .order("created_at", { ascending: true })+ .limit(limit);++ throwIfError(error, "Failed to list running monitor checks");+ return (data ?? []) as MonitorCheckRow[];+}++export async function listMonitorChecks(params: {+ teamId: string;+ monitorId: string;+ limit: number;+ offset: number;+ status?: MonitorCheckRow["status"];+}): Promise<MonitorCheckRow[]> {+ let query = supabase_rr_service+ .from("monitor_checks")+ .select("*")+ .eq("monitor_id", params.monitorId)+ .eq("team_id", params.teamId)+ .order("created_at", { ascending: false });++ if (params.status) {+ query = query.eq("status", params.status);+ }++ const { data, error } = await query.range(+ params.offset,+ params.offset + params.limit - 1,+ );++ throwIfError(error, "Failed to list monitor checks");+ return (data ?? []) as MonitorCheckRow[];+}++export async function updateMonitorCheck(+ checkId: string,+ patch: Partial<MonitorCheckRow>,+): Promise<MonitorCheckRow> {+ const { data, error } = await supabase_service+ .from("monitor_checks")+ .update({+ ...patch,+ updated_at: new Date().toISOString(),+ })+ .eq("id", checkId)+ .select("*")+ .single();++ throwIfError(error, "Failed to update monitor check");+ return data as MonitorCheckRow;+}++export async function insertMonitorCheckPages(+ pages: MonitorCheckPageInsert[],+): Promise<void> {+ if (pages.length === 0) return;++ const { error } = await supabase_service.from("monitor_check_pages").insert(+ pages.map(page => ({+ id: uuidv7(),+ ...page,+ url_hash: page.url_hash ?? hashMonitorUrl(page.url),+ })),+ );++ throwIfError(error, "Failed to insert monitor check pages");+}++export async function listMonitorCheckPages(params: {+ teamId: string;+ monitorId: string;+ checkId: string;+ limit: number;+ skip: number;+ status?: string;+}): Promise<any[]> {+ let query = supabase_rr_service+ .from("monitor_check_pages")+ .select("*")+ .eq("check_id", params.checkId)+ .eq("monitor_id", params.monitorId)+ .eq("team_id", params.teamId)+ .order("created_at", { ascending: true });++ if (params.status) {+ query = query.eq("status", params.status);+ }++ const { data, error } = await query.range(+ params.skip,+ params.skip + params.limit - 1,+ );++ throwIfError(error, "Failed to list monitor check pages");+ return data ?? [];+}++export async function countMonitorCheckPages(params: {+ checkId: string;+ targetId?: string;+ status?: string;+}): Promise<number> {+ let query = supabase_rr_service+ .from("monitor_check_pages")+ .select("id", { count: "exact", head: true })+ .eq("check_id", params.checkId);++ if (params.targetId) {+ query = query.eq("target_id", params.targetId);+ }+ if (params.status) {+ query = query.eq("status", params.status);+ }++ const { count, error } = await query;+ throwIfError(error, "Failed to count monitor check pages");+ return count ?? 0;+}++export async function getMonitorPage(params: {+ monitorId: string;+ targetId: string;+ url: string;+}): Promise<MonitorPageRow | null> {+ const { data, error } = await supabase_rr_service+ .from("monitor_pages")+ .select("*")+ .eq("monitor_id", params.monitorId)+ .eq("target_id", params.targetId)+ .eq("url_hash", hashMonitorUrl(params.url))+ .maybeSingle();++ throwIfError(error, "Failed to get monitor page");+ return data as MonitorPageRow | null;+}++export async function upsertMonitorPage(params: {+ monitorId: string;+ teamId: string;+ targetId: string;+ url: string;+ source: "explicit" | "discovered";+ checkId: string;+ scrapeId: string | null;+ status: "same" | "new" | "changed" | "removed" | "error";+ metadata?: unknown;+}): Promise<void> {+ const now = new Date().toISOString();++ const existing = await getMonitorPage({+ monitorId: params.monitorId,+ targetId: params.targetId,+ url: params.url,+ });++ if (!existing) {+ const { error } = await supabase_service.from("monitor_pages").insert({+ monitor_id: params.monitorId,+ team_id: params.teamId,+ target_id: params.targetId,+ url: params.url,+ url_hash: hashMonitorUrl(params.url),+ source: params.source,+ first_seen_check_id: params.checkId,+ last_seen_check_id:+ params.status === "removed" ? undefined : params.checkId,+ last_changed_check_id:+ params.status === "changed" || params.status === "new"+ ? params.checkId+ : undefined,+ last_scrape_id: params.scrapeId,+ last_status: params.status,+ is_removed: params.status === "removed",+ removed_at: params.status === "removed" ? now : null,+ metadata: params.metadata ?? null,+ created_at: now,+ updated_at: now,+ });++ throwIfError(error, "Failed to insert monitor page");+ return;+ }++ const patch: Record<string, unknown> = {+ last_status: params.status,+ is_removed: params.status === "removed",+ removed_at: params.status === "removed" ? now : null,+ metadata: params.metadata ?? existing.metadata ?? null,+ updated_at: now,+ };+ if (params.status !== "removed") {+ patch.last_seen_check_id = params.checkId;+ patch.last_scrape_id = params.scrapeId;+ }+ if (params.status === "changed" || params.status === "new") {+ patch.last_changed_check_id = params.checkId;+ }++ const { error } = await supabase_service+ .from("monitor_pages")+ .update(patch)+ .eq("id", existing.id);++ throwIfError(error, "Failed to update monitor page");+}++export async function listActiveMonitorPages(params: {+ monitorId: string;+ targetId: string;+}): Promise<MonitorPageRow[]> {+ const pageSize = 1000;+ const pages: MonitorPageRow[] = [];+ let offset = 0;++ while (true) {+ const { data, error } = await supabase_rr_service+ .from("monitor_pages")+ .select("*")+ .eq("monitor_id", params.monitorId)+ .eq("target_id", params.targetId)+ .eq("is_removed", false)+ .order("created_at", { ascending: true })+ .range(offset, offset + pageSize - 1);++ throwIfError(error, "Failed to list active monitor pages");+ const batch = (data ?? []) as MonitorPageRow[];+ pages.push(...batch);+ if (batch.length < pageSize) break;+ offset += pageSize;+ }++ return pages;+}++export async function claimDueMonitors(params: {+ workerId: string;+ limit: number;+ leaseSeconds: number;+}): Promise<MonitorRow[]> {+ const { data, error } = await supabase_service.rpc(+ "monitoring_claim_due_monitors",+ {+ p_worker_id: params.workerId,+ p_limit: params.limit,+ p_lease_seconds: params.leaseSeconds,+ },+ );++ throwIfError(error, "Failed to claim due monitors");+ return (data ?? []) as MonitorRow[];+}diff --git a/apps/api/src/services/monitoring/types.ts b/apps/api/src/services/monitoring/types.tsnew file mode 100644index 000000000..7a1aab521--- /dev/null+++ b/apps/api/src/services/monitoring/types.ts@@ -0,0 +1,275 @@+import { z } from "zod";+import {+ crawlerOptions,+ URL as urlSchema,+ type ScrapeOptions,+} from "../../controllers/v2/types";+import { createWebhookSchema } from "../webhook/schema";+import { parseMonitorScheduleText } from "./cron";++const formatSchema = z.union([z.string(), z.record(z.string(), z.unknown())]);++const scrapeOptionsSchema = z+ .object({+ formats: z.array(formatSchema).optional(),+ })+ .catchall(z.unknown())+ .optional()+ .default({});++const scrapeTargetSchema = z.strictObject({+ id: z.string().uuid().optional(),+ type: z.literal("scrape"),+ urls: z.array(urlSchema).min(1),+ scrapeOptions: scrapeOptionsSchema,+});++const crawlTargetSchema = z.strictObject({+ id: z.string().uuid().optional(),+ type: z.literal("crawl"),+ url: urlSchema,+ crawlOptions: crawlerOptions+ .partial()+ .catchall(z.unknown())+ .optional()+ .default({}),+ scrapeOptions: scrapeOptionsSchema,+});++const monitorTargetSchema = z.union([scrapeTargetSchema, crawlTargetSchema]);++const monitorWebhookSchema = createWebhookSchema([+ "monitor.page",+ "monitor.check.completed",+]);++const monitorScheduleSchema = z+ .strictObject({+ cron: z.string().min(1).max(128).optional(),+ text: z.string().min(1).max(128).optional(),+ timezone: z.string().min(1).max(128).optional().default("UTC"),+ })+ .superRefine((schedule, ctx) => {+ if (!schedule.cron && !schedule.text) {+ ctx.addIssue({+ code: "custom",+ message: "Schedule must include either cron or text",+ path: ["cron"],+ });+ }+ if (schedule.cron && schedule.text) {+ ctx.addIssue({+ code: "custom",+ message: "Schedule must include either cron or text, not both",+ path: ["text"],+ });+ }+ if (schedule.text) {+ try {+ parseMonitorScheduleText(schedule.text);+ } catch (error) {+ ctx.addIssue({+ code: "custom",+ message: error instanceof Error ? error.message : String(error),+ path: ["text"],+ });+ }+ }+ })+ .transform(schedule => ({+ cron: schedule.cron ?? parseMonitorScheduleText(schedule.text!),+ timezone: schedule.timezone,+ }));++const monitorNotificationSchema = z+ .strictObject({+ email: z+ .strictObject({+ enabled: z.boolean().optional().default(false),+ recipients: z.array(z.email()).max(25).optional().default([]),+ includeDiffs: z.boolean().optional().default(false),+ })+ .optional(),+ })+ .optional()+ .default({});++export const createMonitorSchema = z.strictObject({+ name: z.string().min(1).max(256),+ schedule: monitorScheduleSchema,+ webhook: monitorWebhookSchema.optional(),+ notification: monitorNotificationSchema,+ targets: z.array(monitorTargetSchema).min(1).max(50),+ retentionDays: z.number().int().positive().max(365).optional().default(30),+});++export const updateMonitorSchema = createMonitorSchema+ .partial()+ .extend({+ status: z.enum(["active", "paused"]).optional(),+ })+ .refine(x => Object.keys(x).length > 0, "Update body cannot be empty");++export const listMonitorsQuerySchema = z.object({+ limit: z.coerce.number().int().positive().max(100).optional().default(25),+ offset: z.coerce.number().int().nonnegative().optional().default(0),+});++export const listMonitorChecksQuerySchema = z.object({+ limit: z.coerce.number().int().positive().max(100).optional().default(25),+ offset: z.coerce.number().int().nonnegative().optional().default(0),+ status: z+ .enum([+ "queued",+ "running",+ "completed",+ "failed",+ "partial",+ "skipped_overlap",+ ])+ .optional(),+});++export const monitorCheckDetailQuerySchema = z.object({+ limit: z.coerce.number().int().positive().max(100).optional().default(25),+ skip: z.coerce.number().int().nonnegative().optional().default(0),+ status: z.enum(["same", "new", "changed", "removed", "error"]).optional(),+});++export type MonitorTarget = z.infer<typeof monitorTargetSchema> & {+ id: string;+};+export type CreateMonitorRequest = z.infer<typeof createMonitorSchema>;+export type UpdateMonitorRequest = z.infer<typeof updateMonitorSchema>;+type MonitorNotification = z.infer<typeof monitorNotificationSchema>;++export type MonitorRow = {+ id: string;+ team_id: string;+ name: string;+ status: "active" | "paused" | "deleted";+ schedule_cron: string;+ schedule_timezone: string;+ next_run_at: string | null;+ last_run_at: string | null;+ current_check_id: string | null;+ locked_at: string | null;+ locked_until: string | null;+ retention_days: number;+ estimated_credits_per_month: number | null;+ targets: MonitorTarget[];+ webhook: unknown | null;+ notification: MonitorNotification | null;+ last_check_summary: MonitorSummary | null;+ created_at: string;+ updated_at: string;+ deleted_at: string | null;+};++export type MonitorCheckRow = {+ id: string;+ monitor_id: string;+ team_id: string;+ trigger: "scheduled" | "manual";+ status:+ | "queued"+ | "running"+ | "completed"+ | "failed"+ | "partial"+ | "skipped_overlap";+ scheduled_for: string | null;+ started_at: string | null;+ finished_at: string | null;+ estimated_credits: number | null;+ reserved_credits: number | null;+ actual_credits: number | null;+ autumn_lock_id: string | null;+ billing_status:+ | "not_applicable"+ | "reserved"+ | "confirmed"+ | "released"+ | "failed";+ total_pages: number;+ same_count: number;+ changed_count: number;+ new_count: number;+ removed_count: number;+ error_count: number;+ target_results: unknown | null;+ webhook_payload: unknown | null;+ email_payload: unknown | null;+ notification_status: unknown | null;+ error: string | null;+ created_at: string;+ updated_at: string;+};++type MonitorPageStatus = "same" | "new" | "changed" | "removed" | "error";+type MonitorPageSource = "explicit" | "discovered";++export type MonitorPageRow = {+ id: string;+ monitor_id: string;+ team_id: string;+ target_id: string;+ url: string;+ url_hash: string;+ source: MonitorPageSource;+ first_seen_check_id: string | null;+ last_seen_check_id: string | null;+ last_changed_check_id: string | null;+ last_scrape_id: string | null;+ last_status: MonitorPageStatus;+ is_removed: boolean;+ removed_at: string | null;+ metadata: unknown | null;+ created_at: string;+ updated_at: string;+};++export type MonitorSummary = {+ totalPages: number;+ same: number;+ changed: number;+ new: number;+ removed: number;+ error: number;+};++export type MonitorCheckPageInsert = {+ check_id: string;+ monitor_id: string;+ team_id: string;+ target_id: string;+ url: string;+ url_hash?: string;+ status: MonitorPageStatus;+ previous_scrape_id?: string | null;+ current_scrape_id?: string | null;+ diff_gcs_key?: string | null;+ diff_text_bytes?: number | null;+ diff_json_bytes?: number | null;+ status_code?: number | null;+ error?: string | null;+ metadata?: unknown | null;+};++export function withMarkdownFormat(+ options: Record<string, unknown>,+): ScrapeOptions {+ const formats = Array.isArray(options.formats) ? options.formats : [];+ const hasMarkdown = formats.some(format =>+ typeof format === "string"+ ? format === "markdown"+ : typeof format === "object" &&+ format !== null &&+ (format as any).type === "markdown",+ );++ return {+ ...options,+ formats: hasMarkdown ? formats : ["markdown", ...formats],+ } as ScrapeOptions;+}diff --git a/apps/api/src/services/notification/monitoring_email.ts b/apps/api/src/services/notification/monitoring_email.tsnew file mode 100644index 000000000..06d90ed83--- /dev/null+++ b/apps/api/src/services/notification/monitoring_email.ts@@ -0,0 +1,216 @@+import { Resend } from "resend";+import escapeHtml from "escape-html";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";+import { supabase_service } from "../supabase";+import type { MonitorCheckRow, MonitorRow } from "../monitoring/types";++const logger = _logger.child({ module: "monitoring-email" });++type MonitoringEmailPage = {+ url: string;+ status: string;+ error?: string | null;+};++type MonitoringEmailPayload = {+ monitorId: string;+ monitorName: string;+ checkId: string;+ summary: {+ changed: number;+ new: number;+ removed: number;+ error: number;+ totalPages: number;+ };+ pages: MonitoringEmailPage[];+ creditsUsed: number | null;+};++async function getTeamEmails(teamId: string): Promise<string[]> {+ const { data, error } = await supabase_service+ .from("user_teams")+ .select(+ "users(email, id, notification_preferences(unsubscribed_all, email_preferences))",+ )+ .eq("team_id", teamId);++ if (error) {+ logger.warn("Failed to load team emails", { error, teamId });+ return [];+ }++ const emails = new Set<string>();+ for (const row of data ?? []) {+ const user = (row as any).users;+ const email = user?.email;+ if (!email) continue;++ const prefs = Array.isArray(user.notification_preferences)+ ? user.notification_preferences[0]+ : user.notification_preferences;+ if (prefs?.unsubscribed_all) continue;+ if (+ Array.isArray(prefs?.email_preferences) &&+ !prefs.email_preferences.includes("system_alerts")+ ) {+ continue;+ }++ emails.add(email);+ }++ return [...emails];+}++function buildHtml(payload: MonitoringEmailPayload): string {+ const pageItems = payload.pages+ .slice(0, 20)+ .map(page => {+ const url = escapeHtml(page.url);+ return `<li><strong>${escapeHtml(page.status)}</strong>: <a href="${url}">${url}</a>${+ page.error ? ` — ${escapeHtml(page.error)}` : ""+ }</li>`;+ })+ .join("");++ return `Hey there,<br/>+<p>Your Firecrawl monitor <strong>${escapeHtml(payload.monitorName)}</strong> detected activity.</p>+<ul>+ <li>Changed: ${payload.summary.changed}</li>+ <li>New: ${payload.summary.new}</li>+ <li>Removed: ${payload.summary.removed}</li>+ <li>Errors: ${payload.summary.error}</li>+ <li>Total pages checked: ${payload.summary.totalPages}</li>+</ul>+${pageItems ? `<p>Top pages:</p><ul>${pageItems}</ul>` : ""}+<p>Check ID: <code>${escapeHtml(payload.checkId)}</code></p>+<p>Credits used: ${payload.creditsUsed ?? "unknown"}</p>+<br/>Thanks,<br/>Firecrawl Team<br/>`;+}++export async function sendMonitoringEmailSummary(params: {+ monitor: MonitorRow;+ check: MonitorCheckRow;+ pages: MonitoringEmailPage[];+}): Promise<{+ attempted: boolean;+ success: boolean;+ recipients: string[];+ error?: string;+}> {+ const configEmail = params.monitor.notification?.email;+ if (!configEmail?.enabled) {+ logger.info(+ "Skipping monitoring email summary; email notifications disabled",+ {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ },+ );+ return { attempted: false, success: true, recipients: [] };+ }++ if (+ params.check.changed_count ++ params.check.new_count ++ params.check.removed_count ++ params.check.error_count <=+ 0+ ) {+ logger.info("Skipping monitoring email summary; no changes detected", {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ changed: params.check.changed_count,+ new: params.check.new_count,+ removed: params.check.removed_count,+ errors: params.check.error_count,+ });+ return { attempted: false, success: true, recipients: [] };+ }++ const explicitRecipients = configEmail.recipients ?? [];+ const teamRecipients =+ explicitRecipients.length > 0+ ? []+ : await getTeamEmails(params.monitor.team_id);+ const recipients = [...new Set([...explicitRecipients, ...teamRecipients])];+ if (recipients.length === 0) {+ logger.info("Skipping monitoring email summary; no recipients configured", {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ });+ return { attempted: false, success: true, recipients };+ }++ const resendApiKey = config.RESEND_API_KEY?.trim();+ if (!resendApiKey) {+ logger.warn(+ "Skipping monitoring email summary; RESEND_API_KEY is not set",+ {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ recipients,+ },+ );+ return { attempted: false, success: true, recipients };+ }++ const payload: MonitoringEmailPayload = {+ monitorId: params.monitor.id,+ monitorName: params.monitor.name,+ checkId: params.check.id,+ summary: {+ changed: params.check.changed_count,+ new: params.check.new_count,+ removed: params.check.removed_count,+ error: params.check.error_count,+ totalPages: params.check.total_pages,+ },+ pages: params.pages,+ creditsUsed: params.check.actual_credits,+ };++ const resend = new Resend(resendApiKey);+ try {+ const { error } = await resend.emails.send({+ from: "Firecrawl <[email protected]>",+ to: recipients,+ reply_to: "[email protected]",+ subject: `Monitor changes detected: ${params.monitor.name}`,+ html: buildHtml(payload),+ });++ if (error) {+ logger.warn("Monitoring email summary send failed", {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ recipients,+ error,+ });+ return {+ attempted: true,+ success: false,+ recipients,+ error: typeof error === "string" ? error : JSON.stringify(error),+ };+ }++ logger.info("Monitoring email summary sent", {+ monitorId: params.monitor.id,+ checkId: params.check.id,+ recipients,+ });++ return { attempted: true, success: true, recipients };+ } catch (error) {+ logger.warn("Failed to send monitoring email summary", { error });+ return {+ attempted: true,+ success: false,+ recipients,+ error: error instanceof Error ? error.message : String(error),+ };+ }+}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 0afc8fce8..289a544e9 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -25,6 +25,12 @@ import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";import { TransportableError } from "../lib/error";+import {+ processMonitorCheckJob,+ reconcileRunningMonitorChecks,+} from "./monitoring/runner";+import { enqueueDueMonitorChecks } from "./monitoring/scheduler";+import { consumeMonitorCheckJobs } from "./monitoring/queue";configDotenv();@@ -38,6 +44,7 @@ const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;const runningJobs: Set<string> = new Set();+let monitorSchedulerInterval: NodeJS.Timeout | null = null;const processDeepResearchJobInternal = async (token: string,@@ -407,7 +414,7 @@ let currentLiveness: boolean = true;app.get("/liveness", (req, res) => {_logger.info("Liveness endpoint hit");- if (config.USE_DB_AUTHENTICATION) {+ if (config.USE_DB_AUTHENTICATION && config.NUQ_RABBITMQ_URL) {// networking check for Kubernetes environmentsconst host = config.FIRECRAWL_APP_HOST;const port = config.FIRECRAWL_APP_PORT;@@ -452,12 +459,43 @@ app.listen(workerPort, () => {initializeEngineForcing();+ if (config.USE_DB_AUTHENTICATION) {+ monitorSchedulerInterval = setInterval(() => {+ enqueueDueMonitorChecks().catch(error => {+ _logger.error("Failed to enqueue due monitor checks", { error });+ });+ reconcileRunningMonitorChecks().catch(error => {+ _logger.error("Failed to reconcile running monitor checks", { error });+ });+ }, 60_000);+ enqueueDueMonitorChecks().catch(error => {+ _logger.error("Failed to enqueue due monitor checks", { error });+ });+ reconcileRunningMonitorChecks().catch(error => {+ _logger.error("Failed to reconcile running monitor checks", { error });+ });++ await consumeMonitorCheckJobs(processMonitorCheckJob);+ } else if (!config.USE_DB_AUTHENTICATION) {+ _logger.info(+ "Skipping monitor worker startup because database authentication is disabled",+ );+ } else {+ _logger.info(+ "Skipping monitor worker startup because NUQ_RABBITMQ_URL is not configured",+ );+ }+await Promise.all([workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),crawlFinishWorker(),]);+ if (monitorSchedulerInterval) {+ clearInterval(monitorSchedulerInterval);+ }+_logger.info("All workers exited. Waiting for all jobs to finish...");while (runningJobs.size > 0) {diff --git a/apps/api/src/services/webhook/types.ts b/apps/api/src/services/webhook/types.tsindex a3fbdabde..550c4f430 100644--- a/apps/api/src/services/webhook/types.ts+++ b/apps/api/src/services/webhook/types.ts@@ -13,6 +13,8 @@ export enum WebhookEvent {EXTRACT_STARTED = "extract.started",EXTRACT_COMPLETED = "extract.completed",EXTRACT_FAILED = "extract.failed",+ MONITOR_PAGE = "monitor.page",+ MONITOR_CHECK_COMPLETED = "monitor.check.completed",}export type WebhookEventDataMap = {@@ -25,6 +27,8 @@ export type WebhookEventDataMap = {[WebhookEvent.EXTRACT_STARTED]: ExtractStartedData;[WebhookEvent.EXTRACT_COMPLETED]: ExtractCompletedData;[WebhookEvent.EXTRACT_FAILED]: ExtractFailedData;+ [WebhookEvent.MONITOR_PAGE]: MonitorPageData;+ [WebhookEvent.MONITOR_CHECK_COMPLETED]: MonitorCheckCompletedData;};export type WebhookConfig = z.infer<typeof webhookSchema>;@@ -113,3 +117,36 @@ interface ExtractFailedData extends BaseWebhookData {success: false;error: string;}++// monitor+interface MonitorPageData extends BaseWebhookData {+ success: boolean;+ data: {+ monitorId: string;+ checkId: string;+ url: string;+ status: string;+ previousScrapeId?: string | null;+ currentScrapeId?: string | null;+ error?: string | null;+ };+ error?: string;+}++interface MonitorCheckCompletedData extends BaseWebhookData {+ success: boolean;+ data: {+ monitorId: string;+ checkId: string;+ status: string;+ summary: {+ totalPages: number;+ same: number;+ changed: number;+ new: number;+ removed: number;+ error: number;+ };+ };+ error?: string;+}diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 35516137f..d35926142 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -73,6 +73,8 @@ export async function finishCrawlSuper(job: NuQJob<any>) {),zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,cancelled: sc.cancelled ?? false,+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,},false,);@@ -158,6 +160,8 @@ export async function finishCrawlSuper(job: NuQJob<any>) {credits_cost: credits_billed ?? 0,zeroDataRetention: sc.zeroDataRetention || job.data.zeroDataRetention,cancelled: sc.cancelled ?? false,+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,},false,);diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 2abada584..e6ff468da 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -83,6 +83,10 @@ import {import { ScrapeUrlResponse } from "../../scraper/scrapeURL";import { logScrape } from "../logging/log_job";import { FeatureFlag } from "../../scraper/scrapeURL/engines";+import {+ recordMonitorScrapeFailure,+ recordMonitorScrapeSuccess,+} from "../monitoring/results";configDotenv();@@ -456,6 +460,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {v1: job.data.v1,zeroDataRetention: job.data.zeroDataRetention,apiKeyId: job.data.apiKeyId,+ monitoring: job.data.monitoring+ ? { ...job.data.monitoring, source: "discovered" }+ : undefined,},jobId,jobPriority,@@ -529,6 +536,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,skipNuq: job.data.skipNuq ?? false,is_parse: Boolean(job.data.internalOptions?.isParse),+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,},true,);@@ -576,6 +585,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}+ await recordMonitorScrapeSuccess(job, doc).catch(error =>+ logger.warn("Failed to record monitor scrape result", { error }),+ );+logger.debug("Declaring job as done...");await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);} else {@@ -613,6 +626,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,skipNuq: job.data.skipNuq ?? false,is_parse: Boolean(job.data.internalOptions?.isParse),+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,},false,);@@ -630,6 +645,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,}).catch(err => logger.warn("Scrape tracking failed", { error: err }));+ await recordMonitorScrapeSuccess(job, doc).catch(error =>+ logger.warn("Failed to record monitor scrape result", { error }),+ );+if (job.data.skipNuq) {// doesn't use GCS for result retrieval, safe to not awaitlogScrapePromise.catch(err =>@@ -789,6 +808,8 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,skipNuq: job.data.skipNuq ?? false,is_parse: Boolean(job.data.internalOptions?.isParse),+ monitor_id: job.data.monitoring?.monitorId,+ monitor_check_id: job.data.monitoring?.checkId,},true,);@@ -806,6 +827,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,}).catch(err => logger.warn("Scrape tracking failed", { error: err }));+ await recordMonitorScrapeFailure(job, error).catch(err =>+ logger.warn("Failed to record monitor scrape failure", { error: err }),+ );+return data;} finally {if (abortTimeoutHandle) clearTimeout(abortTimeoutHandle);diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex b5d4815fb..8ad6ffbee 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -21,6 +21,12 @@ type ScrapeJobCommon = {traceContext?: SerializedTraceContext;skipNuq?: boolean;requestId?: string;+ monitoring?: {+ monitorId: string;+ checkId: string;+ targetId: string;+ source: "explicit" | "discovered";+ };};export type ScrapeJobData = ScrapeJobCommon &diff --git a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj b/apps/dot-net-sdk/Firecrawl/Firecrawl.csprojindex e2a0f936f..4962ae390 100644--- a/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj+++ b/apps/dot-net-sdk/Firecrawl/Firecrawl.csproj@@ -8,7 +8,7 @@<!-- NuGet package metadata --><PackageId>firecrawl-sdk</PackageId>- <Version>1.2.1</Version>+ <Version>1.3.0</Version><Authors>Firecrawl</Authors><Company>Firecrawl</Company><Description>.NET SDK for the Firecrawl API - web scraping, crawling, and data extraction</Description>diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.csindex 1f2ae2b78..11c5da6a3 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlClient.cs@@ -1,6 +1,7 @@using System.Text.Json;using Firecrawl.Exceptions;using Firecrawl.Models;+using MonitorModel = Firecrawl.Models.Monitor;namespace Firecrawl;@@ -321,6 +322,122 @@ public class FirecrawlClientreturn response.Data ?? throw new FirecrawlException("Map response contained no data");}+ // ================================================================+ // MONITOR+ // ================================================================++ public async Task<MonitorModel> CreateMonitorAsync(+ CreateMonitorRequest request,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(request);++ var response = await _http.PostAsync<ApiResponse<MonitorModel>>(+ "/v2/monitor", request, cancellationToken: cancellationToken);++ return response.Data ?? throw new FirecrawlException("Create monitor response contained no data");+ }++ public async Task<List<MonitorModel>> ListMonitorsAsync(+ int? limit = null,+ int? offset = null,+ CancellationToken cancellationToken = default)+ {+ var response = await _http.GetAsync<ApiResponse<List<MonitorModel>>>(+ $"/v2/monitor{BuildQuery(limit, offset)}", cancellationToken);++ return response.Data ?? new List<MonitorModel>();+ }++ public async Task<MonitorModel> GetMonitorAsync(+ string monitorId,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(monitorId);++ var response = await _http.GetAsync<ApiResponse<MonitorModel>>(+ $"/v2/monitor/{monitorId}", cancellationToken);++ return response.Data ?? throw new FirecrawlException("Get monitor response contained no data");+ }++ public async Task<MonitorModel> UpdateMonitorAsync(+ string monitorId,+ UpdateMonitorRequest request,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(monitorId);+ ArgumentNullException.ThrowIfNull(request);++ var response = await _http.PatchAsync<ApiResponse<MonitorModel>>(+ $"/v2/monitor/{monitorId}", request, cancellationToken);++ return response.Data ?? throw new FirecrawlException("Update monitor response contained no data");+ }++ public async Task<bool> DeleteMonitorAsync(+ string monitorId,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(monitorId);++ var response = await _http.DeleteAsync<Dictionary<string, object>>(+ $"/v2/monitor/{monitorId}", cancellationToken);++ return response.TryGetValue("success", out var success) && success switch+ {+ bool value => value,+ JsonElement element when element.ValueKind == JsonValueKind.True => true,+ _ => false+ };+ }++ public async Task<MonitorCheck> RunMonitorAsync(+ string monitorId,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(monitorId);++ var response = await _http.PostAsync<ApiResponse<MonitorCheck>>(+ $"/v2/monitor/{monitorId}/run", new Dictionary<string, object>(), cancellationToken: cancellationToken);++ return response.Data ?? throw new FirecrawlException("Run monitor response contained no data");+ }++ public async Task<List<MonitorCheck>> ListMonitorChecksAsync(+ string monitorId,+ int? limit = null,+ int? offset = null,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(monitorId);++ var response = await _http.GetAsync<ApiResponse<List<MonitorCheck>>>(+ $"/v2/monitor/{monitorId}/checks{BuildQuery(limit, offset)}", cancellationToken);++ return response.Data ?? new List<MonitorCheck>();+ }++ public async Task<MonitorCheckDetail> GetMonitorCheckAsync(+ string monitorId,+ string checkId,+ int? limit = null,+ int? skip = null,+ string? status = null,+ bool autoPaginate = true,+ CancellationToken cancellationToken = default)+ {+ ArgumentNullException.ThrowIfNull(monitorId);+ ArgumentNullException.ThrowIfNull(checkId);++ var response = await _http.GetAsync<ApiResponse<MonitorCheckDetail>>(+ $"/v2/monitor/{monitorId}/checks/{checkId}{BuildMonitorCheckQuery(limit, skip, status)}",+ cancellationToken);++ var check = response.Data ?? throw new FirecrawlException("Get monitor check response contained no data");+ return autoPaginate ? await PaginateMonitorCheckAsync(check, cancellationToken) : check;+ }+// ================================================================// SEARCH// ================================================================@@ -460,6 +577,32 @@ public class FirecrawlClientreturn job;}+ private async Task<MonitorCheckDetail> PaginateMonitorCheckAsync(+ MonitorCheckDetail check,+ CancellationToken cancellationToken)+ {+ check.Pages ??= new List<MonitorCheckPage>();+ var current = check;++ while (!string.IsNullOrEmpty(current.Next))+ {+ var response = await _http.GetAbsoluteAsync<ApiResponse<MonitorCheckDetail>>(+ current.Next, cancellationToken);+ if (response.Data == null)+ break;++ var nextPage = response.Data;++ if (nextPage.Pages is { Count: > 0 })+ check.Pages.AddRange(nextPage.Pages);++ current = nextPage;+ }++ check.Next = null;+ return check;+ }+// ================================================================// INTERNAL UTILITIES// ================================================================@@ -474,6 +617,32 @@ public class FirecrawlClient?? new Dictionary<string, object>();}+ private static string BuildQuery(int? limit = null, int? offset = null, string? status = null)+ {+ var query = new List<string>();+ if (limit.HasValue)+ query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");+ if (offset.HasValue)+ query.Add($"offset={Uri.EscapeDataString(offset.Value.ToString())}");+ if (!string.IsNullOrWhiteSpace(status))+ query.Add($"status={Uri.EscapeDataString(status)}");++ return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);+ }++ private static string BuildMonitorCheckQuery(int? limit = null, int? skip = null, string? status = null)+ {+ var query = new List<string>();+ if (limit.HasValue)+ query.Add($"limit={Uri.EscapeDataString(limit.Value.ToString())}");+ if (skip.HasValue)+ query.Add($"skip={Uri.EscapeDataString(skip.Value.ToString())}");+ if (!string.IsNullOrWhiteSpace(status))+ query.Add($"status={Uri.EscapeDataString(status)}");++ return query.Count == 0 ? string.Empty : "?" + string.Join("&", query);+ }+private static string ResolveApiKey(string? apiKey){if (!string.IsNullOrWhiteSpace(apiKey))diff --git a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.csindex 2da99b16e..a7ba80dbf 100644--- a/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs+++ b/apps/dot-net-sdk/Firecrawl/FirecrawlHttpClient.cs@@ -139,6 +139,25 @@ internal class FirecrawlHttpClientreturn await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);}+ internal async Task<T> PatchAsync<T>(+ string path,+ object body,+ CancellationToken cancellationToken = default)+ {+ var url = _baseUrl + path;+ var json = JsonSerializer.Serialize(body, JsonOptions);++ HttpRequestMessage BuildRequest()+ {+ var content = new StringContent(json, Encoding.UTF8, "application/json");+ var request = new HttpRequestMessage(HttpMethod.Patch, url) { Content = content };+ ApplyStandardHeaders(request);+ return request;+ }++ return await ExecuteWithRetryAsync<T>(BuildRequest, cancellationToken);+ }+internal async Task<T> GetAbsoluteAsync<T>(string absoluteUrl, CancellationToken cancellationToken = default){// Validate that the pagination URL belongs to the same host to prevent API key exfiltrationdiff --git a/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.csnew file mode 100644index 000000000..ed5c7ef83--- /dev/null+++ b/apps/dot-net-sdk/Firecrawl/Models/MonitorModels.cs@@ -0,0 +1,225 @@+using System.Text.Json.Serialization;++namespace Firecrawl.Models;++public class MonitorSchedule+{+ [JsonPropertyName("cron")]+ public string? Cron { get; set; }++ [JsonPropertyName("timezone")]+ public string? Timezone { get; set; }+}++public class CreateMonitorRequest+{+ [JsonPropertyName("name")]+ public string? Name { get; set; }++ [JsonPropertyName("schedule")]+ public MonitorSchedule? Schedule { get; set; }++ [JsonPropertyName("targets")]+ public List<Dictionary<string, object>>? Targets { get; set; }++ [JsonPropertyName("webhook")]+ public Dictionary<string, object>? Webhook { get; set; }++ [JsonPropertyName("notification")]+ public Dictionary<string, object>? Notification { get; set; }++ [JsonPropertyName("retentionDays")]+ public int? RetentionDays { get; set; }+}++public class UpdateMonitorRequest+{+ [JsonPropertyName("name")]+ public string? Name { get; set; }++ [JsonPropertyName("status")]+ public string? Status { get; set; }++ [JsonPropertyName("schedule")]+ public MonitorSchedule? Schedule { get; set; }++ [JsonPropertyName("targets")]+ public List<Dictionary<string, object>>? Targets { get; set; }++ [JsonPropertyName("webhook")]+ public Dictionary<string, object>? Webhook { get; set; }++ [JsonPropertyName("notification")]+ public Dictionary<string, object>? Notification { get; set; }++ [JsonPropertyName("retentionDays")]+ public int? RetentionDays { get; set; }+}++public class MonitorSummary+{+ [JsonPropertyName("totalPages")]+ public int TotalPages { get; set; }++ [JsonPropertyName("same")]+ public int Same { get; set; }++ [JsonPropertyName("changed")]+ public int Changed { get; set; }++ [JsonPropertyName("new")]+ public int New { get; set; }++ [JsonPropertyName("removed")]+ public int Removed { get; set; }++ [JsonPropertyName("error")]+ public int Error { get; set; }+}++public class Monitor+{+ [JsonPropertyName("id")]+ public string? Id { get; set; }++ [JsonPropertyName("name")]+ public string? Name { get; set; }++ [JsonPropertyName("status")]+ public string? Status { get; set; }++ [JsonPropertyName("schedule")]+ public MonitorSchedule? Schedule { get; set; }++ [JsonPropertyName("nextRunAt")]+ public string? NextRunAt { get; set; }++ [JsonPropertyName("lastRunAt")]+ public string? LastRunAt { get; set; }++ [JsonPropertyName("currentCheckId")]+ public string? CurrentCheckId { get; set; }++ [JsonPropertyName("targets")]+ public List<Dictionary<string, object>>? Targets { get; set; }++ [JsonPropertyName("webhook")]+ public Dictionary<string, object>? Webhook { get; set; }++ [JsonPropertyName("notification")]+ public Dictionary<string, object>? Notification { get; set; }++ [JsonPropertyName("retentionDays")]+ public int RetentionDays { get; set; }++ [JsonPropertyName("estimatedCreditsPerMonth")]+ public int? EstimatedCreditsPerMonth { get; set; }++ [JsonPropertyName("lastCheckSummary")]+ public MonitorSummary? LastCheckSummary { get; set; }++ [JsonPropertyName("createdAt")]+ public string? CreatedAt { get; set; }++ [JsonPropertyName("updatedAt")]+ public string? UpdatedAt { get; set; }+}++public class MonitorCheck+{+ [JsonPropertyName("id")]+ public string? Id { get; set; }++ [JsonPropertyName("monitorId")]+ public string? MonitorId { get; set; }++ [JsonPropertyName("status")]+ public string? Status { get; set; }++ [JsonPropertyName("trigger")]+ public string? Trigger { get; set; }++ [JsonPropertyName("scheduledFor")]+ public string? ScheduledFor { get; set; }++ [JsonPropertyName("startedAt")]+ public string? StartedAt { get; set; }++ [JsonPropertyName("finishedAt")]+ public string? FinishedAt { get; set; }++ [JsonPropertyName("estimatedCredits")]+ public int? EstimatedCredits { get; set; }++ [JsonPropertyName("reservedCredits")]+ public int? ReservedCredits { get; set; }++ [JsonPropertyName("actualCredits")]+ public int? ActualCredits { get; set; }++ [JsonPropertyName("billingStatus")]+ public string? BillingStatus { get; set; }++ [JsonPropertyName("summary")]+ public MonitorSummary? Summary { get; set; }++ [JsonPropertyName("targetResults")]+ public object? TargetResults { get; set; }++ [JsonPropertyName("notificationStatus")]+ public object? NotificationStatus { get; set; }++ [JsonPropertyName("error")]+ public string? Error { get; set; }++ [JsonPropertyName("createdAt")]+ public string? CreatedAt { get; set; }++ [JsonPropertyName("updatedAt")]+ public string? UpdatedAt { get; set; }+}++public class MonitorCheckPage+{+ [JsonPropertyName("id")]+ public string? Id { get; set; }++ [JsonPropertyName("targetId")]+ public string? TargetId { get; set; }++ [JsonPropertyName("url")]+ public string? Url { get; set; }++ [JsonPropertyName("status")]+ public string? Status { get; set; }++ [JsonPropertyName("previousScrapeId")]+ public string? PreviousScrapeId { get; set; }++ [JsonPropertyName("currentScrapeId")]+ public string? CurrentScrapeId { get; set; }++ [JsonPropertyName("statusCode")]+ public int? StatusCode { get; set; }++ [JsonPropertyName("error")]+ public string? Error { get; set; }++ [JsonPropertyName("metadata")]+ public object? Metadata { get; set; }++ [JsonPropertyName("diff")]+ public object? Diff { get; set; }++ [JsonPropertyName("createdAt")]+ public string? CreatedAt { get; set; }+}++public class MonitorCheckDetail : MonitorCheck+{+ [JsonPropertyName("pages")]+ public List<MonitorCheckPage>? Pages { get; set; }++ [JsonPropertyName("next")]+ public string? Next { get; set; }+}diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex d974d853b..3dce580e8 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -1157,4 +1157,274 @@ defmodule Firecrawl doReq.delete!(client(opts), url: "/scrape/#{job_id}/interact")end+ @monitor_key_mapping %{+ name: "name",+ schedule: "schedule",+ targets: "targets",+ webhook: "webhook",+ notification: "notification",+ retention_days: "retentionDays",+ status: "status"+ }++ @monitor_list_key_mapping %{limit: "limit", offset: "offset"}+ @monitor_check_list_key_mapping %{limit: "limit", offset: "offset", status: "status"}+ @monitor_check_key_mapping %{limit: "limit", skip: "skip", status: "status"}++ @monitor_schema NimbleOptions.new!([+ name: [type: :string],+ schedule: [type: :any],+ targets: [type: {:list, :any}],+ webhook: [type: :any],+ notification: [type: :any],+ retention_days: [type: :integer],+ status: [type: :any]+ ])++ @monitor_list_schema NimbleOptions.new!([+ limit: [type: :integer],+ offset: [type: :integer]+ ])++ @monitor_check_list_schema NimbleOptions.new!([+ limit: [type: :integer],+ offset: [type: :integer],+ status: [type: :any]+ ])++ @monitor_check_schema NimbleOptions.new!([+ limit: [type: :integer],+ skip: [type: :integer],+ status: [type: :any]+ ])++ @doc """+ Create a scheduled monitor.++ `POST /monitor`+ """+ @spec create_monitor(keyword(), keyword()) :: response()+ def create_monitor(params \\ [], opts \\ []) do+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_schema) do+ Req.post(client(opts), url: "/monitor", json: to_body(params, @monitor_key_mapping))+ end+ end++ @doc """+ Bang variant of `create_monitor`. Raises on error.+ """+ @spec create_monitor!(keyword(), keyword()) :: Req.Response.t()+ def create_monitor!(params \\ [], opts \\ []) do+ params = NimbleOptions.validate!(params, @monitor_schema)+ Req.post!(client(opts), url: "/monitor", json: to_body(params, @monitor_key_mapping))+ end++ @doc """+ List monitors for the authenticated team.++ `GET /monitor`+ """+ @spec list_monitors(keyword(), keyword()) :: response()+ def list_monitors(params \\ [], opts \\ []) do+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_list_schema) do+ Req.get(client(opts), url: "/monitor", params: to_query(params, @monitor_list_key_mapping))+ end+ end++ @doc """+ Bang variant of `list_monitors`. Raises on error.+ """+ @spec list_monitors!(keyword(), keyword()) :: Req.Response.t()+ def list_monitors!(params \\ [], opts \\ []) do+ params = NimbleOptions.validate!(params, @monitor_list_schema)+ Req.get!(client(opts), url: "/monitor", params: to_query(params, @monitor_list_key_mapping))+ end++ @doc """+ Get a monitor by ID.++ `GET /monitor/{monitorId}`+ """+ @spec get_monitor(String.t(), keyword()) :: response()+ def get_monitor(monitor_id, opts \\ []) do+ Req.get(client(opts), url: "/monitor/#{monitor_id}")+ end++ @doc """+ Bang variant of `get_monitor`. Raises on error.+ """+ @spec get_monitor!(String.t(), keyword()) :: Req.Response.t()+ def get_monitor!(monitor_id, opts \\ []) do+ Req.get!(client(opts), url: "/monitor/#{monitor_id}")+ end++ @doc """+ Update a monitor.++ `PATCH /monitor/{monitorId}`+ """+ @spec update_monitor(String.t(), keyword(), keyword()) :: response()+ def update_monitor(monitor_id, params \\ [], opts \\ []) do+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_schema) do+ Req.patch(client(opts), url: "/monitor/#{monitor_id}", json: to_body(params, @monitor_key_mapping))+ end+ end++ @doc """+ Bang variant of `update_monitor`. Raises on error.+ """+ @spec update_monitor!(String.t(), keyword(), keyword()) :: Req.Response.t()+ def update_monitor!(monitor_id, params \\ [], opts \\ []) do+ params = NimbleOptions.validate!(params, @monitor_schema)+ Req.patch!(client(opts), url: "/monitor/#{monitor_id}", json: to_body(params, @monitor_key_mapping))+ end++ @doc """+ Delete a monitor.++ `DELETE /monitor/{monitorId}`+ """+ @spec delete_monitor(String.t(), keyword()) :: response()+ def delete_monitor(monitor_id, opts \\ []) do+ Req.delete(client(opts), url: "/monitor/#{monitor_id}")+ end++ @doc """+ Bang variant of `delete_monitor`. Raises on error.+ """+ @spec delete_monitor!(String.t(), keyword()) :: Req.Response.t()+ def delete_monitor!(monitor_id, opts \\ []) do+ Req.delete!(client(opts), url: "/monitor/#{monitor_id}")+ end++ @doc """+ Trigger a manual monitor check.++ `POST /monitor/{monitorId}/run`+ """+ @spec run_monitor(String.t(), keyword()) :: response()+ def run_monitor(monitor_id, opts \\ []) do+ Req.post(client(opts), url: "/monitor/#{monitor_id}/run", json: %{})+ end++ @doc """+ Bang variant of `run_monitor`. Raises on error.+ """+ @spec run_monitor!(String.t(), keyword()) :: Req.Response.t()+ def run_monitor!(monitor_id, opts \\ []) do+ Req.post!(client(opts), url: "/monitor/#{monitor_id}/run", json: %{})+ end++ @doc """+ List checks for a monitor.++ `GET /monitor/{monitorId}/checks`+ """+ @spec list_monitor_checks(String.t(), keyword(), keyword()) :: response()+ def list_monitor_checks(monitor_id, params \\ [], opts \\ []) do+ with {:ok, params} <- NimbleOptions.validate(params, @monitor_check_list_schema) do+ Req.get(client(opts),+ url: "/monitor/#{monitor_id}/checks",+ params: to_query(params, @monitor_check_list_key_mapping)+ )+ end+ end++ @doc """+ Bang variant of `list_monitor_checks`. Raises on error.+ """+ @spec list_monitor_checks!(String.t(), keyword(), keyword()) :: Req.Response.t()+ def list_monitor_checks!(monitor_id, params \\ [], opts \\ []) do+ params = NimbleOptions.validate!(params, @monitor_check_list_schema)+ Req.get!(client(opts),+ url: "/monitor/#{monitor_id}/checks",+ params: to_query(params, @monitor_check_list_key_mapping)+ )+ end++ @doc """+ Get a monitor check with paginated page results and inline diffs.++ `GET /monitor/{monitorId}/checks/{checkId}`+ """+ @spec get_monitor_check(String.t(), String.t(), keyword(), keyword()) :: response()+ def get_monitor_check(monitor_id, check_id, params \\ [], opts \\ []) do+ {auto_paginate, opts} = Keyword.pop(opts, :auto_paginate, true)++ with {:ok, params} <- NimbleOptions.validate(params, @monitor_check_schema) do+ case Req.get(client(opts),+ url: "/monitor/#{monitor_id}/checks/#{check_id}",+ params: to_query(params, @monitor_check_key_mapping)+ ) do+ {:ok, response} when auto_paginate -> {:ok, paginate_monitor_check_response(response, opts)}+ result -> result+ end+ end+ end++ @doc """+ Bang variant of `get_monitor_check`. Raises on error.+ """+ @spec get_monitor_check!(String.t(), String.t(), keyword(), keyword()) :: Req.Response.t()+ def get_monitor_check!(monitor_id, check_id, params \\ [], opts \\ []) do+ {auto_paginate, opts} = Keyword.pop(opts, :auto_paginate, true)+ params = NimbleOptions.validate!(params, @monitor_check_schema)+ response = Req.get!(client(opts),+ url: "/monitor/#{monitor_id}/checks/#{check_id}",+ params: to_query(params, @monitor_check_key_mapping)+ )++ if auto_paginate, do: paginate_monitor_check_response!(response, opts), else: response+ end++ defp paginate_monitor_check_response(%Req.Response{body: body} = response, opts) when is_map(body) do+ data = Map.get(body, "data") || %{}+ pages = Map.get(data, "pages") || []+ next = Map.get(body, "next") || Map.get(data, "next")+ pages = fetch_monitor_check_pages(next, pages, opts)+ data = data |> Map.put("pages", pages) |> Map.put("next", nil)+ %{response | body: body |> Map.put("data", data) |> Map.put("next", nil)}+ end++ defp paginate_monitor_check_response(response, _opts), do: response++ defp paginate_monitor_check_response!(%Req.Response{body: body} = response, opts) when is_map(body) do+ data = Map.get(body, "data") || %{}+ pages = Map.get(data, "pages") || []+ next = Map.get(body, "next") || Map.get(data, "next")+ pages = fetch_monitor_check_pages!(next, pages, opts)+ data = data |> Map.put("pages", pages) |> Map.put("next", nil)+ %{response | body: body |> Map.put("data", data) |> Map.put("next", nil)}+ end++ defp paginate_monitor_check_response!(response, _opts), do: response++ defp fetch_monitor_check_pages(nil, pages, _opts), do: pages+ defp fetch_monitor_check_pages("", pages, _opts), do: pages++ defp fetch_monitor_check_pages(next, pages, opts) do+ case Req.get(client(opts), url: next) do+ {:ok, %Req.Response{body: body}} when is_map(body) ->+ data = Map.get(body, "data") || %{}+ next_pages = Map.get(data, "pages") || []+ next_url = Map.get(body, "next") || Map.get(data, "next")+ fetch_monitor_check_pages(next_url, pages ++ next_pages, opts)++ _ ->+ pages+ end+ end++ defp fetch_monitor_check_pages!(nil, pages, _opts), do: pages+ defp fetch_monitor_check_pages!("", pages, _opts), do: pages++ defp fetch_monitor_check_pages!(next, pages, opts) do+ response = Req.get!(client(opts), url: next)+ body = response.body+ data = if is_map(body), do: Map.get(body, "data") || %{}, else: %{}+ next_pages = Map.get(data, "pages") || []+ next_url = if is_map(body), do: Map.get(body, "next") || Map.get(data, "next"), else: nil+ fetch_monitor_check_pages!(next_url, pages ++ next_pages, opts)+ end+enddiff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex 479cf2217..e15468af2 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.2.1"+ @version "1.3.0"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/go-sdk/firecrawl.go b/apps/go-sdk/firecrawl.goindex 33f0361f7..57f608e0c 100644--- a/apps/go-sdk/firecrawl.go+++ b/apps/go-sdk/firecrawl.go@@ -16,6 +16,7 @@ import ("encoding/json""fmt""net/http"+ "net/url""os""strings""time"@@ -24,7 +25,7 @@ import ()const (- defaultPollInterval = 2 // seconds+ defaultPollInterval = 2 // secondsdefaultJobTimeout = 300 // seconds)@@ -383,6 +384,145 @@ func (c *Client) Map(ctx context.Context, url string, opts *MapOptions) (*MapDatreturn data, nil}+// ================================================================+// MONITOR+// ================================================================++// CreateMonitor creates a scheduled monitor.+func (c *Client) CreateMonitor(ctx context.Context, req *MonitorCreateRequest) (*Monitor, error) {+ if req == nil {+ return nil, &FirecrawlError{Message: "monitor request is required"}+ }+ raw, err := c.http.post(ctx, "/v2/monitor", req, nil)+ if err != nil {+ return nil, err+ }+ return extractDataAs[Monitor](raw)+}++// ListMonitors lists monitors for the authenticated team.+func (c *Client) ListMonitors(ctx context.Context, opts *ListMonitorsOptions) ([]Monitor, error) {+ path := "/v2/monitor" + listQuery(opts)+ raw, err := c.http.get(ctx, path)+ if err != nil {+ return nil, err+ }+ data, err := extractDataAs[[]Monitor](raw)+ if err != nil {+ return nil, err+ }+ return *data, nil+}++// GetMonitor gets a monitor by ID.+func (c *Client) GetMonitor(ctx context.Context, monitorID string) (*Monitor, error) {+ if monitorID == "" {+ return nil, &FirecrawlError{Message: "monitor ID is required"}+ }+ raw, err := c.http.get(ctx, "/v2/monitor/"+monitorID)+ if err != nil {+ return nil, err+ }+ return extractDataAs[Monitor](raw)+}++// UpdateMonitor updates a monitor.+func (c *Client) UpdateMonitor(ctx context.Context, monitorID string, req *MonitorUpdateRequest) (*Monitor, error) {+ if monitorID == "" {+ return nil, &FirecrawlError{Message: "monitor ID is required"}+ }+ if req == nil {+ return nil, &FirecrawlError{Message: "monitor update request is required"}+ }+ raw, err := c.http.patch(ctx, "/v2/monitor/"+monitorID, req)+ if err != nil {+ return nil, err+ }+ return extractDataAs[Monitor](raw)+}++// DeleteMonitor deletes a monitor.+func (c *Client) DeleteMonitor(ctx context.Context, monitorID string) (bool, error) {+ if monitorID == "" {+ return false, &FirecrawlError{Message: "monitor ID is required"}+ }+ raw, err := c.http.delete(ctx, "/v2/monitor/"+monitorID)+ if err != nil {+ return false, err+ }+ var resp struct {+ Success bool `json:"success"`+ }+ if err := json.Unmarshal(raw, &resp); err != nil {+ return false, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ return resp.Success, nil+}++// RunMonitor triggers a manual monitor check.+func (c *Client) RunMonitor(ctx context.Context, monitorID string) (*MonitorCheck, error) {+ if monitorID == "" {+ return nil, &FirecrawlError{Message: "monitor ID is required"}+ }+ raw, err := c.http.post(ctx, "/v2/monitor/"+monitorID+"/run", map[string]interface{}{}, nil)+ if err != nil {+ return nil, err+ }+ return extractDataAs[MonitorCheck](raw)+}++// ListMonitorChecks lists checks for a monitor.+func (c *Client) ListMonitorChecks(ctx context.Context, monitorID string, opts *ListMonitorChecksOptions) ([]MonitorCheck, error) {+ if monitorID == "" {+ return nil, &FirecrawlError{Message: "monitor ID is required"}+ }+ path := "/v2/monitor/" + monitorID + "/checks" + checksQuery(opts)+ raw, err := c.http.get(ctx, path)+ if err != nil {+ return nil, err+ }+ data, err := extractDataAs[[]MonitorCheck](raw)+ if err != nil {+ return nil, err+ }+ return *data, nil+}++// GetMonitorCheck gets a monitor check with page results, auto-paginated by default.+func (c *Client) GetMonitorCheck(ctx context.Context, monitorID, checkID string, opts *GetMonitorCheckOptions) (*MonitorCheckDetail, error) {+ if monitorID == "" || checkID == "" {+ return nil, &FirecrawlError{Message: "monitor ID and check ID are required"}+ }+ path := "/v2/monitor/" + monitorID + "/checks/" + checkID + monitorCheckPageQuery(opts)+ raw, err := c.http.get(ctx, path)+ if err != nil {+ return nil, err+ }+ detail, err := extractMonitorCheckDetail(raw)+ if err != nil {+ return nil, err+ }++ autoPaginate := true+ if opts != nil && opts.AutoPaginate != nil {+ autoPaginate = *opts.AutoPaginate+ }+ for autoPaginate && detail.Next != "" {+ raw, err := c.http.getAbsolute(ctx, detail.Next)+ if err != nil {+ return nil, err+ }+ nextPage, err := extractMonitorCheckDetail(raw)+ if err != nil {+ return nil, err+ }+ detail.Pages = append(detail.Pages, nextPage.Pages...)+ detail.Next = nextPage.Next+ }++ return detail, nil+}+// ================================================================// SEARCH// ================================================================@@ -793,6 +933,63 @@ func mergeOptions(body map[string]interface{}, opts interface{}) {}}+func listQuery(opts *ListMonitorsOptions) string {+ if opts == nil {+ return ""+ }+ values := url.Values{}+ if opts.Limit != nil {+ values.Set("limit", fmt.Sprintf("%d", *opts.Limit))+ }+ if opts.Offset != nil {+ values.Set("offset", fmt.Sprintf("%d", *opts.Offset))+ }+ if encoded := values.Encode(); encoded != "" {+ return "?" + encoded+ }+ return ""+}++func checksQuery(opts *ListMonitorChecksOptions) string {+ if opts == nil {+ return ""+ }+ values := url.Values{}+ if opts.Limit != nil {+ values.Set("limit", fmt.Sprintf("%d", *opts.Limit))+ }+ if opts.Offset != nil {+ values.Set("offset", fmt.Sprintf("%d", *opts.Offset))+ }+ if opts.Status != "" {+ values.Set("status", opts.Status)+ }+ if encoded := values.Encode(); encoded != "" {+ return "?" + encoded+ }+ return ""+}++func monitorCheckPageQuery(opts *GetMonitorCheckOptions) string {+ if opts == nil {+ return ""+ }+ values := url.Values{}+ if opts.Limit != nil {+ values.Set("limit", fmt.Sprintf("%d", *opts.Limit))+ }+ if opts.Skip != nil {+ values.Set("skip", fmt.Sprintf("%d", *opts.Skip))+ }+ if opts.Status != "" {+ values.Set("status", opts.Status)+ }+ if encoded := values.Encode(); encoded != "" {+ return "?" + encoded+ }+ return ""+}+// extractDataAs extracts the "data" field from a raw API response and deserializes it.func extractDataAs[T any](raw json.RawMessage) (*T, error) {var envelope map[string]json.RawMessage@@ -816,3 +1013,17 @@ func extractDataAs[T any](raw json.RawMessage) (*T, error) {}return &result, nil}++func extractMonitorCheckDetail(raw json.RawMessage) (*MonitorCheckDetail, error) {+ var envelope struct {+ Data MonitorCheckDetail `json:"data"`+ Next string `json:"next"`+ }+ if err := json.Unmarshal(raw, &envelope); err != nil {+ return nil, &FirecrawlError{Message: fmt.Sprintf("failed to decode response: %v", err)}+ }+ if envelope.Next != "" {+ envelope.Data.Next = envelope.Next+ }+ return &envelope.Data, nil+}diff --git a/apps/go-sdk/http_client.go b/apps/go-sdk/http_client.goindex 99156559b..e17f221d6 100644--- a/apps/go-sdk/http_client.go+++ b/apps/go-sdk/http_client.go@@ -49,6 +49,12 @@ func (h *httpClient) post(ctx context.Context, path string, body interface{}, exreturn h.doJSON(ctx, "POST", url, body, extraHeaders)}+// patch sends a PATCH request.+func (h *httpClient) patch(ctx context.Context, path string, body interface{}) (json.RawMessage, error) {+ url := h.baseURL + path+ return h.doJSON(ctx, "PATCH", url, body, nil)+}+// get sends a GET request.func (h *httpClient) get(ctx context.Context, path string) (json.RawMessage, error) {url := h.baseURL + pathdiff --git a/apps/go-sdk/models.go b/apps/go-sdk/models.goindex d4335d74e..5877eadea 100644--- a/apps/go-sdk/models.go+++ b/apps/go-sdk/models.go@@ -100,6 +100,126 @@ type MapData struct {Links []LinkResult `json:"links,omitempty"`}+// MonitorSchedule configures when a monitor runs.+type MonitorSchedule struct {+ Cron string `json:"cron"`+ Timezone string `json:"timezone,omitempty"`+}++// MonitorCreateRequest creates a scheduled monitor.+type MonitorCreateRequest struct {+ Name string `json:"name"`+ Schedule MonitorSchedule `json:"schedule"`+ Targets []map[string]interface{} `json:"targets"`+ Webhook map[string]interface{} `json:"webhook,omitempty"`+ Notification map[string]interface{} `json:"notification,omitempty"`+ RetentionDays *int `json:"retentionDays,omitempty"`+}++// MonitorUpdateRequest updates a scheduled monitor.+type MonitorUpdateRequest struct {+ Name string `json:"name,omitempty"`+ Status string `json:"status,omitempty"`+ Schedule *MonitorSchedule `json:"schedule,omitempty"`+ Targets []map[string]interface{} `json:"targets,omitempty"`+ Webhook map[string]interface{} `json:"webhook,omitempty"`+ Notification map[string]interface{} `json:"notification,omitempty"`+ RetentionDays *int `json:"retentionDays,omitempty"`+}++// Monitor represents a scheduled monitor.+type Monitor struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Status string `json:"status"`+ Schedule MonitorSchedule `json:"schedule"`+ NextRunAt string `json:"nextRunAt,omitempty"`+ LastRunAt string `json:"lastRunAt,omitempty"`+ CurrentCheckID string `json:"currentCheckId,omitempty"`+ Targets []map[string]interface{} `json:"targets,omitempty"`+ Webhook map[string]interface{} `json:"webhook,omitempty"`+ Notification map[string]interface{} `json:"notification,omitempty"`+ RetentionDays int `json:"retentionDays"`+ EstimatedCreditsPerMonth *int `json:"estimatedCreditsPerMonth,omitempty"`+ LastCheckSummary *MonitorSummary `json:"lastCheckSummary,omitempty"`+ CreatedAt string `json:"createdAt,omitempty"`+ UpdatedAt string `json:"updatedAt,omitempty"`+}++// MonitorSummary summarizes page statuses in a check.+type MonitorSummary struct {+ TotalPages int `json:"totalPages"`+ Same int `json:"same"`+ Changed int `json:"changed"`+ New int `json:"new"`+ Removed int `json:"removed"`+ Error int `json:"error"`+}++// MonitorCheck represents a single monitor run.+type MonitorCheck struct {+ ID string `json:"id"`+ MonitorID string `json:"monitorId"`+ Status string `json:"status"`+ Trigger string `json:"trigger"`+ ScheduledFor string `json:"scheduledFor,omitempty"`+ StartedAt string `json:"startedAt,omitempty"`+ FinishedAt string `json:"finishedAt,omitempty"`+ EstimatedCredits *int `json:"estimatedCredits,omitempty"`+ ReservedCredits *int `json:"reservedCredits,omitempty"`+ ActualCredits *int `json:"actualCredits,omitempty"`+ BillingStatus string `json:"billingStatus,omitempty"`+ Summary MonitorSummary `json:"summary"`+ TargetResults interface{} `json:"targetResults,omitempty"`+ NotificationStatus interface{} `json:"notificationStatus,omitempty"`+ Error string `json:"error,omitempty"`+ CreatedAt string `json:"createdAt,omitempty"`+ UpdatedAt string `json:"updatedAt,omitempty"`+}++// MonitorCheckPage is a single page result in a monitor check.+type MonitorCheckPage struct {+ ID string `json:"id"`+ TargetID string `json:"targetId"`+ URL string `json:"url"`+ Status string `json:"status"`+ PreviousScrapeID string `json:"previousScrapeId,omitempty"`+ CurrentScrapeID string `json:"currentScrapeId,omitempty"`+ StatusCode *int `json:"statusCode,omitempty"`+ Error string `json:"error,omitempty"`+ Metadata interface{} `json:"metadata,omitempty"`+ Diff interface{} `json:"diff,omitempty"`+ CreatedAt string `json:"createdAt,omitempty"`+}++// MonitorCheckDetail includes paginated page results and inline diffs.+type MonitorCheckDetail struct {+ MonitorCheck+ Pages []MonitorCheckPage `json:"pages,omitempty"`+ Next string `json:"next,omitempty"`+}++// ListMonitorsOptions controls monitor list pagination.+type ListMonitorsOptions struct {+ Limit *int+ Offset *int+}++// ListMonitorChecksOptions controls monitor check pagination/filtering.+type ListMonitorChecksOptions struct {+ Limit *int+ Offset *int+ Status string+}++// GetMonitorCheckOptions controls monitor check page pagination/filtering.+type GetMonitorCheckOptions struct {+ Limit *int+ Skip *int+ Status string+ AutoPaginate *bool+}+// SearchData represents the result of a search request.type SearchData struct {Web []map[string]interface{} `json:"web,omitempty"`@@ -142,13 +262,13 @@ type BrowserCreateResponse struct {// BrowserExecuteResponse is returned when executing code in a browser session.type BrowserExecuteResponse struct {- Success bool `json:"success"`- Stdout string `json:"stdout,omitempty"`- Result string `json:"result,omitempty"`- Stderr string `json:"stderr,omitempty"`- ExitCode *int `json:"exitCode,omitempty"`- Killed *bool `json:"killed,omitempty"`- Error string `json:"error,omitempty"`+ Success bool `json:"success"`+ Stdout string `json:"stdout,omitempty"`+ Result string `json:"result,omitempty"`+ Stderr string `json:"stderr,omitempty"`+ ExitCode *int `json:"exitCode,omitempty"`+ Killed *bool `json:"killed,omitempty"`+ Error string `json:"error,omitempty"`}// BrowserDeleteResponse is returned when deleting a browser session.diff --git a/apps/java-sdk/build.gradle.kts b/apps/java-sdk/build.gradle.ktsindex 3183e7110..3479bb1e7 100644--- a/apps/java-sdk/build.gradle.kts+++ b/apps/java-sdk/build.gradle.kts@@ -4,7 +4,7 @@ plugins {}group = "com.firecrawl"-version = "1.4.1"+version = "1.5.0"java {sourceCompatibility = JavaVersion.VERSION_11diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.javaindex 9055be3e7..bfca8dfc0 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlClient.java@@ -446,6 +446,81 @@ public class FirecrawlClient {return extractData(http.post("/v2/map", body, Map.class), MapData.class);}+ // ================================================================+ // MONITOR+ // ================================================================++ public Monitor createMonitor(Map<String, Object> request) {+ Objects.requireNonNull(request, "Monitor request is required");+ return extractData(http.post("/v2/monitor", request, Map.class), Monitor.class);+ }++ public List<Monitor> listMonitors() {+ return listMonitors(null, null);+ }++ public List<Monitor> listMonitors(Integer limit, Integer offset) {+ Map raw = http.get("/v2/monitor" + listQuery(limit, offset), Map.class);+ return extractDataList(raw, Monitor.class);+ }++ public Monitor getMonitor(String monitorId) {+ Objects.requireNonNull(monitorId, "Monitor ID is required");+ return extractData(http.get("/v2/monitor/" + monitorId, Map.class), Monitor.class);+ }++ public Monitor updateMonitor(String monitorId, Map<String, Object> request) {+ Objects.requireNonNull(monitorId, "Monitor ID is required");+ Objects.requireNonNull(request, "Monitor update request is required");+ return extractData(http.patch("/v2/monitor/" + monitorId, request, Map.class), Monitor.class);+ }++ @SuppressWarnings("unchecked")+ public boolean deleteMonitor(String monitorId) {+ Objects.requireNonNull(monitorId, "Monitor ID is required");+ Map<String, Object> response = http.delete("/v2/monitor/" + monitorId, Map.class);+ return Boolean.TRUE.equals(response.get("success"));+ }++ public MonitorCheck runMonitor(String monitorId) {+ Objects.requireNonNull(monitorId, "Monitor ID is required");+ return extractData(+ http.post("/v2/monitor/" + monitorId + "/run", Collections.emptyMap(), Map.class),+ MonitorCheck.class+ );+ }++ public List<MonitorCheck> listMonitorChecks(String monitorId) {+ return listMonitorChecks(monitorId, null, null);+ }++ public List<MonitorCheck> listMonitorChecks(String monitorId, Integer limit, Integer offset) {+ Objects.requireNonNull(monitorId, "Monitor ID is required");+ Map raw = http.get("/v2/monitor/" + monitorId + "/checks" + listQuery(limit, offset), Map.class);+ return extractDataList(raw, MonitorCheck.class);+ }++ public MonitorCheckDetail getMonitorCheck(String monitorId, String checkId) {+ return getMonitorCheck(monitorId, checkId, null, null, null, true);+ }++ public MonitorCheckDetail getMonitorCheck(+ String monitorId, String checkId, Integer limit, Integer skip, String status) {+ return getMonitorCheck(monitorId, checkId, limit, skip, status, true);+ }++ public MonitorCheckDetail getMonitorCheck(+ String monitorId, String checkId, Integer limit, Integer skip, String status, boolean autoPaginate) {+ Objects.requireNonNull(monitorId, "Monitor ID is required");+ Objects.requireNonNull(checkId, "Check ID is required");+ MonitorCheckDetail check = extractData(+ http.get("/v2/monitor/" + monitorId + "/checks/" + checkId+ + monitorCheckQuery(limit, skip, status), Map.class),+ MonitorCheckDetail.class+ );+ return autoPaginate ? paginateMonitorCheck(check) : check;+ }+// ================================================================// SEARCH// ================================================================@@ -845,6 +920,50 @@ public class FirecrawlClient {return CompletableFuture.supplyAsync(() -> map(url, options), asyncExecutor);}+ public CompletableFuture<Monitor> createMonitorAsync(Map<String, Object> request) {+ return CompletableFuture.supplyAsync(() -> createMonitor(request), asyncExecutor);+ }++ public CompletableFuture<List<Monitor>> listMonitorsAsync(Integer limit, Integer offset) {+ return CompletableFuture.supplyAsync(() -> listMonitors(limit, offset), asyncExecutor);+ }++ public CompletableFuture<Monitor> getMonitorAsync(String monitorId) {+ return CompletableFuture.supplyAsync(() -> getMonitor(monitorId), asyncExecutor);+ }++ public CompletableFuture<Monitor> updateMonitorAsync(String monitorId, Map<String, Object> request) {+ return CompletableFuture.supplyAsync(() -> updateMonitor(monitorId, request), asyncExecutor);+ }++ public CompletableFuture<Boolean> deleteMonitorAsync(String monitorId) {+ return CompletableFuture.supplyAsync(() -> deleteMonitor(monitorId), asyncExecutor);+ }++ public CompletableFuture<MonitorCheck> runMonitorAsync(String monitorId) {+ return CompletableFuture.supplyAsync(() -> runMonitor(monitorId), asyncExecutor);+ }++ public CompletableFuture<List<MonitorCheck>> listMonitorChecksAsync(String monitorId, Integer limit, Integer offset) {+ return CompletableFuture.supplyAsync(() -> listMonitorChecks(monitorId, limit, offset), asyncExecutor);+ }++ public CompletableFuture<MonitorCheckDetail> getMonitorCheckAsync(+ String monitorId, String checkId, Integer limit, Integer skip, String status) {+ return CompletableFuture.supplyAsync(+ () -> getMonitorCheck(monitorId, checkId, limit, skip, status),+ asyncExecutor+ );+ }++ public CompletableFuture<MonitorCheckDetail> getMonitorCheckAsync(+ String monitorId, String checkId, Integer limit, Integer skip, String status, boolean autoPaginate) {+ return CompletableFuture.supplyAsync(+ () -> getMonitorCheck(monitorId, checkId, limit, skip, status, autoPaginate),+ asyncExecutor+ );+ }+/*** Asynchronously runs an agent task and waits for completion.*@@ -1076,6 +1195,25 @@ public class FirecrawlClient {return job;}+ private MonitorCheckDetail paginateMonitorCheck(MonitorCheckDetail check) {+ if (check.getPages() == null) {+ check.setPages(new ArrayList<>());+ }+ MonitorCheckDetail current = check;+ while (current.getNext() != null && !current.getNext().isEmpty()) {+ MonitorCheckDetail nextPage = extractData(+ http.getAbsolute(current.getNext(), Map.class),+ MonitorCheckDetail.class+ );+ if (nextPage.getPages() != null && !nextPage.getPages().isEmpty()) {+ check.getPages().addAll(nextPage.getPages());+ }+ current = nextPage;+ }+ check.setNext(null);+ return check;+ }+// ================================================================// INTERNAL UTILITIES// ================================================================@@ -1093,6 +1231,35 @@ public class FirecrawlClient {return http.objectMapper.convertValue(data, type);}+ private <T> List<T> extractDataList(Map rawResponse, Class<T> type) {+ Object data = rawResponse.get("data");+ if (!(data instanceof List<?>)) {+ return Collections.emptyList();+ }+ List<T> result = new ArrayList<>();+ for (Object item : (List<?>) data) {+ result.add(http.objectMapper.convertValue(item, type));+ }+ return result;+ }++ private String listQuery(Integer limit, Integer offset) {+ List<String> parts = new ArrayList<>();+ if (limit != null) parts.add("limit=" + limit);+ if (offset != null) parts.add("offset=" + offset);+ return parts.isEmpty() ? "" : "?" + String.join("&", parts);+ }++ private String monitorCheckQuery(Integer limit, Integer skip, String status) {+ List<String> parts = new ArrayList<>();+ if (limit != null) parts.add("limit=" + limit);+ if (skip != null) parts.add("skip=" + skip);+ if (status != null && !status.isBlank()) {+ parts.add("status=" + java.net.URLEncoder.encode(status, java.nio.charset.StandardCharsets.UTF_8));+ }+ return parts.isEmpty() ? "" : "?" + String.join("&", parts);+ }+/*** Merges a typed options object into a request body map, using Jackson serialization.*/diff --git a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.javaindex 33e1fe5d4..695c9e079 100644--- a/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java+++ b/apps/java-sdk/src/main/java/com/firecrawl/client/FirecrawlHttpClient.java@@ -86,6 +86,27 @@ class FirecrawlHttpClient {return executeWithRetry(request, responseType);}+ /**+ * Sends a PATCH request with JSON body.+ */+ <T> T patch(String path, Object body, Class<T> responseType) {+ String url = baseUrl + path;+ String json;+ try {+ json = objectMapper.writeValueAsString(body);+ } catch (JsonProcessingException e) {+ throw new FirecrawlException("Failed to serialize request body", e);+ }+ RequestBody requestBody = RequestBody.create(json, JSON);+ Request request = new Request.Builder()+ .url(url)+ .header("Authorization", "Bearer " + apiKey)+ .header("Content-Type", "application/json")+ .patch(requestBody)+ .build();+ return executeWithRetry(request, responseType);+ }+/*** Sends a POST multipart/form-data request.*/diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/Monitor.java b/apps/java-sdk/src/main/java/com/firecrawl/models/Monitor.javanew file mode 100644index 000000000..c881fb9f9--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/Monitor.java@@ -0,0 +1,40 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import java.util.List;+import java.util.Map;++@JsonIgnoreProperties(ignoreUnknown = true)+public class Monitor {+ private String id;+ private String name;+ private String status;+ private MonitorSchedule schedule;+ private String nextRunAt;+ private String lastRunAt;+ private String currentCheckId;+ private List<Map<String, Object>> targets;+ private Map<String, Object> webhook;+ private Map<String, Object> notification;+ private int retentionDays;+ private Integer estimatedCreditsPerMonth;+ private MonitorSummary lastCheckSummary;+ private String createdAt;+ private String updatedAt;++ public String getId() { return id; }+ public String getName() { return name; }+ public String getStatus() { return status; }+ public MonitorSchedule getSchedule() { return schedule; }+ public String getNextRunAt() { return nextRunAt; }+ public String getLastRunAt() { return lastRunAt; }+ public String getCurrentCheckId() { return currentCheckId; }+ public List<Map<String, Object>> getTargets() { return targets; }+ public Map<String, Object> getWebhook() { return webhook; }+ public Map<String, Object> getNotification() { return notification; }+ public int getRetentionDays() { return retentionDays; }+ public Integer getEstimatedCreditsPerMonth() { return estimatedCreditsPerMonth; }+ public MonitorSummary getLastCheckSummary() { return lastCheckSummary; }+ public String getCreatedAt() { return createdAt; }+ public String getUpdatedAt() { return updatedAt; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheck.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheck.javanew file mode 100644index 000000000..4b3f8212e--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheck.java@@ -0,0 +1,42 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorCheck {+ private String id;+ private String monitorId;+ private String status;+ private String trigger;+ private String scheduledFor;+ private String startedAt;+ private String finishedAt;+ private Integer estimatedCredits;+ private Integer reservedCredits;+ private Integer actualCredits;+ private String billingStatus;+ private MonitorSummary summary;+ private Object targetResults;+ private Object notificationStatus;+ private String error;+ private String createdAt;+ private String updatedAt;++ public String getId() { return id; }+ public String getMonitorId() { return monitorId; }+ public String getStatus() { return status; }+ public String getTrigger() { return trigger; }+ public String getScheduledFor() { return scheduledFor; }+ public String getStartedAt() { return startedAt; }+ public String getFinishedAt() { return finishedAt; }+ public Integer getEstimatedCredits() { return estimatedCredits; }+ public Integer getReservedCredits() { return reservedCredits; }+ public Integer getActualCredits() { return actualCredits; }+ public String getBillingStatus() { return billingStatus; }+ public MonitorSummary getSummary() { return summary; }+ public Object getTargetResults() { return targetResults; }+ public Object getNotificationStatus() { return notificationStatus; }+ public String getError() { return error; }+ public String getCreatedAt() { return createdAt; }+ public String getUpdatedAt() { return updatedAt; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckDetail.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckDetail.javanew file mode 100644index 000000000..cdb3c710d--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckDetail.java@@ -0,0 +1,15 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;+import java.util.List;++@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorCheckDetail extends MonitorCheck {+ private List<MonitorCheckPage> pages;+ private String next;++ public List<MonitorCheckPage> getPages() { return pages; }+ public void setPages(List<MonitorCheckPage> pages) { this.pages = pages; }+ public String getNext() { return next; }+ public void setNext(String next) { this.next = next; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.javanew file mode 100644index 000000000..231d3788e--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorCheckPage.java@@ -0,0 +1,30 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorCheckPage {+ private String id;+ private String targetId;+ private String url;+ private String status;+ private String previousScrapeId;+ private String currentScrapeId;+ private Integer statusCode;+ private String error;+ private Object metadata;+ private Object diff;+ private String createdAt;++ public String getId() { return id; }+ public String getTargetId() { return targetId; }+ public String getUrl() { return url; }+ public String getStatus() { return status; }+ public String getPreviousScrapeId() { return previousScrapeId; }+ public String getCurrentScrapeId() { return currentScrapeId; }+ public Integer getStatusCode() { return statusCode; }+ public String getError() { return error; }+ public Object getMetadata() { return metadata; }+ public Object getDiff() { return diff; }+ public String getCreatedAt() { return createdAt; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSchedule.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSchedule.javanew file mode 100644index 000000000..b8e3ff19f--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSchedule.java@@ -0,0 +1,12 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorSchedule {+ private String cron;+ private String timezone;++ public String getCron() { return cron; }+ public String getTimezone() { return timezone; }+}diff --git a/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSummary.java b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSummary.javanew file mode 100644index 000000000..3f04548fc--- /dev/null+++ b/apps/java-sdk/src/main/java/com/firecrawl/models/MonitorSummary.java@@ -0,0 +1,25 @@+package com.firecrawl.models;++import com.fasterxml.jackson.annotation.JsonIgnoreProperties;++@JsonIgnoreProperties(ignoreUnknown = true)+public class MonitorSummary {+ private int totalPages;+ private int same;+ private int changed;+ private int newCount;+ private int removed;+ private int error;++ public int getTotalPages() { return totalPages; }+ public int getSame() { return same; }+ public int getChanged() { return changed; }+ public int getNew() { return newCount; }+ public int getRemoved() { return removed; }+ public int getError() { return error; }++ @com.fasterxml.jackson.annotation.JsonProperty("new")+ private void setNewCount(int value) {+ this.newCount = value;+ }+}diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 51f8f4ace..f88ddd72b 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.21.1",+ "version": "4.22.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.tsindex 20e2afe42..32e823197 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/pagination.test.ts@@ -1,6 +1,7 @@import { describe, test, expect, jest } from "@jest/globals";import { getCrawlStatus } from "../../../v2/methods/crawl";import { getBatchScrapeStatus } from "../../../v2/methods/batch";+import { getMonitorCheck } from "../../../v2/methods/monitor";describe("JS SDK v2 pagination", () => {function makeHttp(getImpl: (url: string) => any) {@@ -64,6 +65,26 @@ describe("JS SDK v2 pagination", () => {expect(res.next).toBe("https://api/nextBatch");});+ test("monitor check: default autoPaginate aggregates pages and nulls next", async () => {+ const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };+ const second = { status: 200, data: { success: true, next: null, data: { pages: [{ url: "b", status: "same" }], next: null } } };+ const http = makeHttp((url) => {+ if (url.includes("/v2/monitor/")) return first;+ return second;+ });+ const res = await getMonitorCheck(http, "mon1", "check1");+ expect(res.pages.length).toBe(2);+ expect(res.next).toBeNull();+ });++ test("monitor check: autoPaginate=false returns next", async () => {+ const first = { status: 200, data: { success: true, next: "https://api/m1", data: { id: "check1", monitorId: "mon1", status: "completed", trigger: "manual", billingStatus: "confirmed", summary: {}, createdAt: "now", updatedAt: "now", pages: [{ url: "a", status: "changed" }], next: "https://api/m1" } } };+ const http = makeHttp(() => first);+ const res = await getMonitorCheck(http, "mon1", "check1", { autoPaginate: false });+ expect(res.pages.length).toBe(1);+ expect(res.next).toBe("https://api/m1");+ });+test("crawl: maxWaitTime stops pagination after first page", async () => {const first = { status: 200, data: { success: true, status: "completed", completed: 1, total: 5, next: "https://api/n1", data: [{ markdown: "a" }] } };const p1 = { status: 200, data: { success: true, next: "https://api/n2", data: [{ markdown: "b" }] } };diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 613a28f7f..f375bb457 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -32,6 +32,16 @@ import {listBrowsers,} from "./methods/browser";import { getConcurrency, getCreditUsage, getQueueStatus, getTokenUsage, getCreditUsageHistorical, getTokenUsageHistorical } from "./methods/usage";+import {+ createMonitor as createMonitorMethod,+ deleteMonitor as deleteMonitorMethod,+ getMonitor as getMonitorMethod,+ getMonitorCheck as getMonitorCheckMethod,+ listMonitorChecks as listMonitorChecksMethod,+ listMonitors as listMonitorsMethod,+ runMonitor as runMonitorMethod,+ updateMonitor as updateMonitorMethod,+} from "./methods/monitor";import type {Document,ParseFile,@@ -60,6 +70,14 @@ import type {ScrapeExecuteRequest,ScrapeExecuteResponse,ScrapeBrowserDeleteResponse,+ CreateMonitorRequest,+ ListMonitorChecksOptions,+ ListMonitorsOptions,+ Monitor,+ MonitorCheck,+ MonitorCheckDetail,+ GetMonitorCheckOptions,+ UpdateMonitorRequest,} from "./types";import { Watcher } from "./watcher";import type { WatcherOptions } from "./watcher";@@ -276,6 +294,73 @@ export class FirecrawlClient {return crawlParamsPreview(this.http, url, prompt);}+ // Monitor+ /**+ * Create a scheduled monitor.+ */+ async createMonitor(request: CreateMonitorRequest): Promise<Monitor> {+ return createMonitorMethod(this.http, request);+ }++ /**+ * List monitors for the authenticated team.+ */+ async listMonitors(options?: ListMonitorsOptions): Promise<Monitor[]> {+ return listMonitorsMethod(this.http, options);+ }++ /**+ * Get a monitor by id.+ */+ async getMonitor(monitorId: string): Promise<Monitor> {+ return getMonitorMethod(this.http, monitorId);+ }++ /**+ * Update a monitor.+ */+ async updateMonitor(+ monitorId: string,+ request: UpdateMonitorRequest,+ ): Promise<Monitor> {+ return updateMonitorMethod(this.http, monitorId, request);+ }++ /**+ * Delete a monitor.+ */+ async deleteMonitor(monitorId: string): Promise<boolean> {+ return deleteMonitorMethod(this.http, monitorId);+ }++ /**+ * Trigger a manual monitor check.+ */+ async runMonitor(monitorId: string): Promise<MonitorCheck> {+ return runMonitorMethod(this.http, monitorId);+ }++ /**+ * List checks for a monitor.+ */+ async listMonitorChecks(+ monitorId: string,+ options?: ListMonitorChecksOptions,+ ): Promise<MonitorCheck[]> {+ return listMonitorChecksMethod(this.http, monitorId, options);+ }++ /**+ * Get a monitor check with paginated page results and inline diffs.+ */+ async getMonitorCheck(+ monitorId: string,+ checkId: string,+ options?: GetMonitorCheckOptions,+ ): Promise<MonitorCheckDetail> {+ return getMonitorCheckMethod(this.http, monitorId, checkId, options);+ }+// Batch/*** Start a batch scrape job for multiple URLs (async).diff --git a/apps/js-sdk/firecrawl/src/v2/methods/monitor.ts b/apps/js-sdk/firecrawl/src/v2/methods/monitor.tsnew file mode 100644index 000000000..3e4221da8--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/monitor.ts@@ -0,0 +1,181 @@+import {+ type CreateMonitorRequest,+ type ListMonitorsOptions,+ type ListMonitorChecksOptions,+ type Monitor,+ type MonitorCheck,+ type MonitorCheckDetail,+ type MonitorCheckPage,+ type GetMonitorCheckOptions,+ type UpdateMonitorRequest,+} from "../types";+import { HttpClient } from "../utils/httpClient";+import {+ throwForBadResponse,+ normalizeAxiosError,+} from "../utils/errorHandler";+import { fetchAllPages } from "../utils/pagination";++type ApiResponse<T> = {+ success: boolean;+ data?: T;+ id?: string;+ error?: string;+};++function queryString(params?: Record<string, unknown>): string {+ if (!params) return "";+ const query = new URLSearchParams();+ for (const [key, value] of Object.entries(params)) {+ if (value !== undefined && value !== null) query.set(key, String(value));+ }+ const str = query.toString();+ return str ? `?${str}` : "";+}++function dataOrThrow<T>(res: { status: number; data?: ApiResponse<T> }, action: string): T {+ if (res.status !== 200 || !res.data?.success || res.data.data == null) {+ throwForBadResponse(res as any, action);+ }+ return res.data.data;+}++export async function createMonitor(+ http: HttpClient,+ request: CreateMonitorRequest,+): Promise<Monitor> {+ try {+ const res = await http.post<ApiResponse<Monitor>>("/v2/monitor", request as any);+ return dataOrThrow(res, "create monitor");+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "create monitor");+ throw err;+ }+}++export async function listMonitors(+ http: HttpClient,+ options?: ListMonitorsOptions,+): Promise<Monitor[]> {+ try {+ const res = await http.get<ApiResponse<Monitor[]>>(+ `/v2/monitor${queryString(options as Record<string, unknown>)}`,+ );+ return dataOrThrow(res, "list monitors");+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "list monitors");+ throw err;+ }+}++export async function getMonitor(+ http: HttpClient,+ monitorId: string,+): Promise<Monitor> {+ try {+ const res = await http.get<ApiResponse<Monitor>>(`/v2/monitor/${monitorId}`);+ return dataOrThrow(res, "get monitor");+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get monitor");+ throw err;+ }+}++export async function updateMonitor(+ http: HttpClient,+ monitorId: string,+ request: UpdateMonitorRequest,+): Promise<Monitor> {+ try {+ const res = await http.patch<ApiResponse<Monitor>>(+ `/v2/monitor/${monitorId}`,+ request as any,+ );+ return dataOrThrow(res, "update monitor");+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "update monitor");+ throw err;+ }+}++export async function deleteMonitor(+ http: HttpClient,+ monitorId: string,+): Promise<boolean> {+ try {+ const res = await http.delete<ApiResponse<unknown>>(`/v2/monitor/${monitorId}`);+ if (res.status !== 200 || !res.data?.success) {+ throwForBadResponse(res, "delete monitor");+ }+ return true;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "delete monitor");+ throw err;+ }+}++export async function runMonitor(+ http: HttpClient,+ monitorId: string,+): Promise<MonitorCheck> {+ try {+ const res = await http.post<ApiResponse<MonitorCheck>>(+ `/v2/monitor/${monitorId}/run`,+ {},+ );+ return dataOrThrow(res, "run monitor");+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "run monitor");+ throw err;+ }+}++export async function listMonitorChecks(+ http: HttpClient,+ monitorId: string,+ options?: ListMonitorChecksOptions,+): Promise<MonitorCheck[]> {+ try {+ const res = await http.get<ApiResponse<MonitorCheck[]>>(+ `/v2/monitor/${monitorId}/checks${queryString(options as Record<string, unknown>)}`,+ );+ return dataOrThrow(res, "list monitor checks");+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "list monitor checks");+ throw err;+ }+}++export async function getMonitorCheck(+ http: HttpClient,+ monitorId: string,+ checkId: string,+ options?: GetMonitorCheckOptions,+): Promise<MonitorCheckDetail> {+ try {+ const { autoPaginate: _autoPaginate, maxPages: _maxPages, maxResults: _maxResults, maxWaitTime: _maxWaitTime, ...query } = options ?? {};+ const res = await http.get<ApiResponse<MonitorCheckDetail>>(+ `/v2/monitor/${monitorId}/checks/${checkId}${queryString(query as Record<string, unknown>)}`,+ );+ const detail = dataOrThrow(res, "get monitor check");+ const next = res.data?.next ?? detail.next ?? null;+ const auto = options?.autoPaginate ?? true;+ if (!auto || !next) {+ return { ...detail, next };+ }++ return {+ ...detail,+ pages: await fetchAllPages<MonitorCheckPage>(+ http,+ next,+ detail.pages || [],+ options,+ ),+ next: null,+ };+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get monitor check");+ throw err;+ }+}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex f72a6b8ea..a07232dc9 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -606,6 +606,154 @@ export interface MapOptions {location?: LocationConfig;}+export interface MonitorSchedule {+ cron: string;+ timezone?: string;+}++export interface MonitorEmailNotification {+ enabled?: boolean;+ recipients?: string[];+ includeDiffs?: boolean;+}++export interface MonitorNotification {+ email?: MonitorEmailNotification;+}++export interface MonitorWebhookConfig {+ url: string;+ headers?: Record<string, string>;+ metadata?: Record<string, string>;+ events?: string[];+}++export interface MonitorScrapeTarget {+ id?: string;+ type: 'scrape';+ urls: string[];+ scrapeOptions?: ScrapeOptions;+}++export interface MonitorCrawlTarget {+ id?: string;+ type: 'crawl';+ url: string;+ crawlOptions?: CrawlOptions;+ scrapeOptions?: ScrapeOptions;+}++export type MonitorTarget = MonitorScrapeTarget | MonitorCrawlTarget;++export interface CreateMonitorRequest {+ name: string;+ schedule: MonitorSchedule;+ webhook?: MonitorWebhookConfig;+ notification?: MonitorNotification;+ targets: MonitorTarget[];+ retentionDays?: number;+}++export interface UpdateMonitorRequest {+ name?: string;+ status?: 'active' | 'paused';+ schedule?: MonitorSchedule;+ webhook?: MonitorWebhookConfig | null;+ notification?: MonitorNotification | null;+ targets?: MonitorTarget[];+ retentionDays?: number;+}++export interface MonitorSummary {+ totalPages: number;+ same: number;+ changed: number;+ new: number;+ removed: number;+ error: number;+}++export interface Monitor {+ id: string;+ name: string;+ status: 'active' | 'paused' | 'deleted';+ schedule: MonitorSchedule;+ nextRunAt?: string | null;+ lastRunAt?: string | null;+ currentCheckId?: string | null;+ targets: MonitorTarget[];+ webhook?: MonitorWebhookConfig | null;+ notification?: MonitorNotification | null;+ retentionDays: number;+ estimatedCreditsPerMonth?: number | null;+ lastCheckSummary?: MonitorSummary | null;+ createdAt: string;+ updatedAt: string;+}++export interface MonitorCheck {+ id: string;+ monitorId: string;+ status:+ | 'queued'+ | 'running'+ | 'completed'+ | 'failed'+ | 'partial'+ | 'skipped_overlap';+ trigger: 'scheduled' | 'manual';+ scheduledFor?: string | null;+ startedAt?: string | null;+ finishedAt?: string | null;+ estimatedCredits?: number | null;+ reservedCredits?: number | null;+ actualCredits?: number | null;+ billingStatus:+ | 'not_applicable'+ | 'reserved'+ | 'confirmed'+ | 'released'+ | 'failed';+ summary: MonitorSummary;+ targetResults?: unknown;+ notificationStatus?: unknown;+ error?: string | null;+ createdAt: string;+ updatedAt: string;+}++export interface MonitorCheckPage {+ id: string;+ targetId: string;+ url: string;+ status: 'same' | 'new' | 'changed' | 'removed' | 'error';+ previousScrapeId?: string | null;+ currentScrapeId?: string | null;+ statusCode?: number | null;+ error?: string | null;+ metadata?: unknown;+ diff?: unknown;+ createdAt: string;+}++export interface MonitorCheckDetail extends MonitorCheck {+ pages: MonitorCheckPage[];+ next?: string | null;+}++export interface ListMonitorsOptions {+ limit?: number;+ offset?: number;+}++export type ListMonitorChecksOptions = ListMonitorsOptions;++export type GetMonitorCheckOptions = PaginationConfig & {+ limit?: number;+ skip?: number;+ status?: MonitorCheckPage["status"];+};+export interface ExtractResponse {success?: boolean;id?: string;diff --git a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.tsindex 5dc2cbbdb..842a0ea25 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/httpClient.ts@@ -149,6 +149,20 @@ export class HttpClient {return this.request<T>({ method: "delete", url: endpoint, headers });}+ patch<T = any>(+ endpoint: string,+ body: Record<string, unknown>,+ options?: RequestOptions,+ ) {+ return this.request<T>({+ method: "patch",+ url: endpoint,+ data: body,+ headers: options?.headers,+ timeout: options?.timeoutMs,+ });+ }+prepareHeaders(idempotencyKey?: string): Record<string, string> {const headers: Record<string, string> = {};if (idempotencyKey) headers["x-idempotency-key"] = idempotencyKey;diff --git a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts b/apps/js-sdk/firecrawl/src/v2/utils/pagination.tsindex 4c54d1271..cb8273cfc 100644--- a/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts+++ b/apps/js-sdk/firecrawl/src/v2/utils/pagination.ts@@ -2,14 +2,14 @@ import type { HttpClient } from "../utils/httpClient";import type { Document, PaginationConfig } from "../types";/**- * Shared helper to follow `next` cursors and aggregate documents with limits.+ * Shared helper to follow `next` URLs and aggregate paginated result arrays.*/-export async function fetchAllPages(+export async function fetchAllPages<T = Document>(http: HttpClient,nextUrl: string,- initial: Document[],+ initial: T[],pagination?: PaginationConfig-): Promise<Document[]> {+): Promise<T[]> {const docs = initial.slice();let current: string | null = nextUrl;let pageCount = 0;@@ -22,21 +22,24 @@ export async function fetchAllPages(if (maxPages != null && pageCount >= maxPages) break;if (maxWaitTime != null && (Date.now() - started) / 1000 > maxWaitTime) break;- let payload: { success: boolean; next?: string | null; data?: Document[] } | null = null;+ let payload: { success: boolean; next?: string | null; data?: T[] | { pages?: T[]; next?: string | null } } | null = null;try {- const res = await http.get<{ success: boolean; next?: string | null; data?: Document[] }>(current);+ const res = await http.get<{ success: boolean; next?: string | null; data?: T[] | { pages?: T[]; next?: string | null } }>(current);payload = res.data;} catch {break; // axios rejects on non-2xx; stop pagination gracefully}if (!payload?.success) break;- for (const d of payload.data || []) {+ const pageData = Array.isArray(payload.data)+ ? payload.data+ : payload.data?.pages || [];+ for (const d of pageData) {if (maxResults != null && docs.length >= maxResults) break;- docs.push(d as Document);+ docs.push(d as T);}if (maxResults != null && docs.length >= maxResults) break;- current = (payload.next ?? null) as string | null;+ current = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;pageCount += 1;}return docs;diff --git a/apps/php-sdk/src/Client/FirecrawlClient.php b/apps/php-sdk/src/Client/FirecrawlClient.phpindex e237b51c1..23dbab668 100644--- a/apps/php-sdk/src/Client/FirecrawlClient.php+++ b/apps/php-sdk/src/Client/FirecrawlClient.php@@ -24,6 +24,9 @@ use Firecrawl\Models\CreditUsage;use Firecrawl\Models\Document;use Firecrawl\Models\MapData;use Firecrawl\Models\MapOptions;+use Firecrawl\Models\Monitor;+use Firecrawl\Models\MonitorCheck;+use Firecrawl\Models\MonitorCheckDetail;use Firecrawl\Models\ParseFile;use Firecrawl\Models\ParseOptions;use Firecrawl\Models\ScrapeOptions;@@ -316,6 +319,142 @@ final class FirecrawlClientreturn MapData::fromArray($response['data'] ?? $response);}+ // ================================================================+ // MONITOR+ // ================================================================++ /**+ * Create a scheduled monitor.+ *+ * @param array<string, mixed> $schedule+ * @param list<array<string, mixed>> $targets+ * @param array<string, mixed>|null $webhook+ * @param array<string, mixed>|null $notification+ */+ public function createMonitor(+ string $name,+ array $schedule,+ array $targets,+ ?array $webhook = null,+ ?array $notification = null,+ ?int $retentionDays = null,+ ): Monitor {+ $body = array_filter([+ 'name' => $name,+ 'schedule' => $schedule,+ 'targets' => $targets,+ 'webhook' => $webhook,+ 'notification' => $notification,+ 'retentionDays' => $retentionDays,+ ], static fn ($value) => $value !== null);++ $response = $this->http->post('/v2/monitor', $body);++ return Monitor::fromArray($response['data'] ?? $response);+ }++ /**+ * @return list<Monitor>+ */+ public function listMonitors(?int $limit = null, ?int $offset = null): array+ {+ $response = $this->http->get('/v2/monitor' . $this->query([+ 'limit' => $limit,+ 'offset' => $offset,+ ]));++ return array_map(+ static fn (array $item): Monitor => Monitor::fromArray($item),+ $response['data'] ?? [],+ );+ }++ public function getMonitor(string $monitorId): Monitor+ {+ $response = $this->http->get("/v2/monitor/{$monitorId}");++ return Monitor::fromArray($response['data'] ?? $response);+ }++ /**+ * @param array<string, mixed> $attributes+ */+ public function updateMonitor(string $monitorId, array $attributes): Monitor+ {+ $response = $this->http->patch("/v2/monitor/{$monitorId}", $attributes);++ return Monitor::fromArray($response['data'] ?? $response);+ }++ public function deleteMonitor(string $monitorId): bool+ {+ $response = $this->http->delete("/v2/monitor/{$monitorId}");++ return ($response['success'] ?? false) === true;+ }++ public function runMonitor(string $monitorId): MonitorCheck+ {+ $response = $this->http->post("/v2/monitor/{$monitorId}/run", []);++ return MonitorCheck::fromArray($response['data'] ?? $response);+ }++ /**+ * @return list<MonitorCheck>+ */+ public function listMonitorChecks(string $monitorId, ?int $limit = null, ?int $offset = null): array+ {+ $response = $this->http->get("/v2/monitor/{$monitorId}/checks" . $this->query([+ 'limit' => $limit,+ 'offset' => $offset,+ ]));++ return array_map(+ static fn (array $item): MonitorCheck => MonitorCheck::fromArray($item),+ $response['data'] ?? [],+ );+ }++ public function getMonitorCheck(+ string $monitorId,+ string $checkId,+ ?int $limit = null,+ ?int $skip = null,+ ?string $status = null,+ bool $autoPaginate = true,+ ): MonitorCheckDetail {+ $response = $this->http->get("/v2/monitor/{$monitorId}/checks/{$checkId}" . $this->query([+ 'limit' => $limit,+ 'skip' => $skip,+ 'status' => $status,+ ]));++ $data = $response['data'] ?? $response;+ if (isset($response['next'])) {+ $data['next'] = $response['next'];+ }++ if (!$autoPaginate) {+ return MonitorCheckDetail::fromArray($data);+ }++ while (isset($data['next']) && is_string($data['next']) && $data['next'] !== '') {+ $this->assertSameOrigin($data['next']);+ $nextResponse = $this->http->getAbsolute($data['next']);+ $nextData = $nextResponse['data'] ?? $nextResponse;+ if (isset($nextResponse['next'])) {+ $nextData['next'] = $nextResponse['next'];+ }++ $data['pages'] = array_merge($data['pages'] ?? [], $nextData['pages'] ?? []);+ $data['next'] = $nextData['next'] ?? null;+ }++ $data['next'] = null;+ return MonitorCheckDetail::fromArray($data);+ }+// ================================================================// SEARCH// ================================================================@@ -505,6 +644,16 @@ final class FirecrawlClient}}+ /**+ * @param array<string, scalar|null> $params+ */+ private function query(array $params): string+ {+ $params = array_filter($params, static fn ($value) => $value !== null && $value !== '');++ return $params === [] ? '' : '?' . http_build_query($params);+ }+private function pollCrawl(?string $jobId,int $pollIntervalSec,@@ -553,10 +702,29 @@ final class FirecrawlClientprivate function assertSameOrigin(string $url): void{+ $baseScheme = parse_url($this->http->getBaseUrl(), PHP_URL_SCHEME);$baseHost = parse_url($this->http->getBaseUrl(), PHP_URL_HOST);+ $basePort = parse_url($this->http->getBaseUrl(), PHP_URL_PORT);+ $nextScheme = parse_url($url, PHP_URL_SCHEME);$nextHost = parse_url($url, PHP_URL_HOST);-- if ($baseHost === null || $nextHost === null || strcasecmp($baseHost, $nextHost) !== 0) {+ $nextPort = parse_url($url, PHP_URL_PORT);++ $basePort ??= is_string($baseScheme) && strcasecmp($baseScheme, 'https') === 0+ ? 443+ : 80;+ $nextPort ??= is_string($nextScheme) && strcasecmp($nextScheme, 'https') === 0+ ? 443+ : 80;++ if (+ $baseScheme === null ||+ $nextScheme === null ||+ $baseHost === null ||+ $nextHost === null ||+ strcasecmp($baseScheme, $nextScheme) !== 0 ||+ strcasecmp($baseHost, $nextHost) !== 0 ||+ $basePort !== $nextPort+ ) {throw new FirecrawlException('Pagination URL origin does not match the API base URL. Refusing to follow: ' . $url,);diff --git a/apps/php-sdk/src/Client/FirecrawlHttpClient.php b/apps/php-sdk/src/Client/FirecrawlHttpClient.phpindex b878b74e8..fa1d35f5d 100644--- a/apps/php-sdk/src/Client/FirecrawlHttpClient.php+++ b/apps/php-sdk/src/Client/FirecrawlHttpClient.php@@ -67,6 +67,15 @@ final class FirecrawlHttpClientreturn $this->request('DELETE', $this->baseUrl . $path);}+ /**+ * @param array<string, mixed> $body+ * @return array<string, mixed>+ */+ public function patch(string $path, array $body): array+ {+ return $this->request('PATCH', $this->baseUrl . $path, $body);+ }+/*** Send a POST request with a multipart/form-data body.*@@ -145,7 +154,7 @@ final class FirecrawlHttpClientif ($multipart !== null) {$options[RequestOptions::MULTIPART] = $multipart;- } elseif ($method === 'POST' && $body !== []) {+ } elseif (($method === 'POST' || $method === 'PATCH') && $body !== []) {$options[RequestOptions::JSON] = $body;}diff --git a/apps/php-sdk/src/Models/Monitor.php b/apps/php-sdk/src/Models/Monitor.phpnew file mode 100644index 000000000..fe4a94d04--- /dev/null+++ b/apps/php-sdk/src/Models/Monitor.php@@ -0,0 +1,76 @@+<?php++declare(strict_types=1);++namespace Firecrawl\Models;++final class Monitor+{+ /**+ * @param list<array<string, mixed>> $targets+ * @param array<string, mixed>|null $schedule+ * @param array<string, mixed>|null $webhook+ * @param array<string, mixed>|null $notification+ * @param array<string, mixed>|null $lastCheckSummary+ */+ public function __construct(+ private readonly ?string $id = null,+ private readonly ?string $name = null,+ private readonly ?string $status = null,+ private readonly ?array $schedule = null,+ private readonly ?string $nextRunAt = null,+ private readonly ?string $lastRunAt = null,+ private readonly ?string $currentCheckId = null,+ private readonly array $targets = [],+ private readonly ?array $webhook = null,+ private readonly ?array $notification = null,+ private readonly ?int $retentionDays = null,+ private readonly ?int $estimatedCreditsPerMonth = null,+ private readonly ?array $lastCheckSummary = null,+ private readonly ?string $createdAt = null,+ private readonly ?string $updatedAt = null,+ ) {}++ /** @param array<string, mixed> $data */+ public static function fromArray(array $data): self+ {+ return new self(+ id: isset($data['id']) ? (string) $data['id'] : null,+ name: isset($data['name']) ? (string) $data['name'] : null,+ status: isset($data['status']) ? (string) $data['status'] : null,+ schedule: isset($data['schedule']) && is_array($data['schedule']) ? $data['schedule'] : null,+ nextRunAt: isset($data['nextRunAt']) ? (string) $data['nextRunAt'] : null,+ lastRunAt: isset($data['lastRunAt']) ? (string) $data['lastRunAt'] : null,+ currentCheckId: isset($data['currentCheckId']) ? (string) $data['currentCheckId'] : null,+ targets: isset($data['targets']) && is_array($data['targets']) ? $data['targets'] : [],+ webhook: isset($data['webhook']) && is_array($data['webhook']) ? $data['webhook'] : null,+ notification: isset($data['notification']) && is_array($data['notification']) ? $data['notification'] : null,+ retentionDays: isset($data['retentionDays']) ? (int) $data['retentionDays'] : null,+ estimatedCreditsPerMonth: isset($data['estimatedCreditsPerMonth']) ? (int) $data['estimatedCreditsPerMonth'] : null,+ lastCheckSummary: isset($data['lastCheckSummary']) && is_array($data['lastCheckSummary']) ? $data['lastCheckSummary'] : null,+ createdAt: isset($data['createdAt']) ? (string) $data['createdAt'] : null,+ updatedAt: isset($data['updatedAt']) ? (string) $data['updatedAt'] : null,+ );+ }++ public function getId(): ?string { return $this->id; }+ public function getName(): ?string { return $this->name; }+ public function getStatus(): ?string { return $this->status; }+ /** @return array<string, mixed>|null */+ public function getSchedule(): ?array { return $this->schedule; }+ public function getNextRunAt(): ?string { return $this->nextRunAt; }+ public function getLastRunAt(): ?string { return $this->lastRunAt; }+ public function getCurrentCheckId(): ?string { return $this->currentCheckId; }+ /** @return list<array<string, mixed>> */+ public function getTargets(): array { return $this->targets; }+ /** @return array<string, mixed>|null */+ public function getWebhook(): ?array { return $this->webhook; }+ /** @return array<string, mixed>|null */+ public function getNotification(): ?array { return $this->notification; }+ public function getRetentionDays(): ?int { return $this->retentionDays; }+ public function getEstimatedCreditsPerMonth(): ?int { return $this->estimatedCreditsPerMonth; }+ /** @return array<string, mixed>|null */+ public function getLastCheckSummary(): ?array { return $this->lastCheckSummary; }+ public function getCreatedAt(): ?string { return $this->createdAt; }+ public function getUpdatedAt(): ?string { return $this->updatedAt; }+}diff --git a/apps/php-sdk/src/Models/MonitorCheck.php b/apps/php-sdk/src/Models/MonitorCheck.phpnew file mode 100644index 000000000..935f20c04--- /dev/null+++ b/apps/php-sdk/src/Models/MonitorCheck.php@@ -0,0 +1,77 @@+<?php++declare(strict_types=1);++namespace Firecrawl\Models;++/** @phpstan-consistent-constructor */+class MonitorCheck+{+ /**+ * @param array<string, mixed> $summary+ * @param mixed $targetResults+ * @param mixed $notificationStatus+ */+ public function __construct(+ private readonly ?string $id = null,+ private readonly ?string $monitorId = null,+ private readonly ?string $status = null,+ private readonly ?string $trigger = null,+ private readonly ?string $scheduledFor = null,+ private readonly ?string $startedAt = null,+ private readonly ?string $finishedAt = null,+ private readonly ?int $estimatedCredits = null,+ private readonly ?int $reservedCredits = null,+ private readonly ?int $actualCredits = null,+ private readonly ?string $billingStatus = null,+ private readonly array $summary = [],+ private readonly mixed $targetResults = null,+ private readonly mixed $notificationStatus = null,+ private readonly ?string $error = null,+ private readonly ?string $createdAt = null,+ private readonly ?string $updatedAt = null,+ ) {}++ /** @param array<string, mixed> $data */+ public static function fromArray(array $data): static+ {+ return new static(+ id: isset($data['id']) ? (string) $data['id'] : null,+ monitorId: isset($data['monitorId']) ? (string) $data['monitorId'] : null,+ status: isset($data['status']) ? (string) $data['status'] : null,+ trigger: isset($data['trigger']) ? (string) $data['trigger'] : null,+ scheduledFor: isset($data['scheduledFor']) ? (string) $data['scheduledFor'] : null,+ startedAt: isset($data['startedAt']) ? (string) $data['startedAt'] : null,+ finishedAt: isset($data['finishedAt']) ? (string) $data['finishedAt'] : null,+ estimatedCredits: isset($data['estimatedCredits']) ? (int) $data['estimatedCredits'] : null,+ reservedCredits: isset($data['reservedCredits']) ? (int) $data['reservedCredits'] : null,+ actualCredits: isset($data['actualCredits']) ? (int) $data['actualCredits'] : null,+ billingStatus: isset($data['billingStatus']) ? (string) $data['billingStatus'] : null,+ summary: isset($data['summary']) && is_array($data['summary']) ? $data['summary'] : [],+ targetResults: $data['targetResults'] ?? null,+ notificationStatus: $data['notificationStatus'] ?? null,+ error: isset($data['error']) ? (string) $data['error'] : null,+ createdAt: isset($data['createdAt']) ? (string) $data['createdAt'] : null,+ updatedAt: isset($data['updatedAt']) ? (string) $data['updatedAt'] : null,+ );+ }++ public function getId(): ?string { return $this->id; }+ public function getMonitorId(): ?string { return $this->monitorId; }+ public function getStatus(): ?string { return $this->status; }+ public function getTrigger(): ?string { return $this->trigger; }+ public function getScheduledFor(): ?string { return $this->scheduledFor; }+ public function getStartedAt(): ?string { return $this->startedAt; }+ public function getFinishedAt(): ?string { return $this->finishedAt; }+ public function getEstimatedCredits(): ?int { return $this->estimatedCredits; }+ public function getReservedCredits(): ?int { return $this->reservedCredits; }+ public function getActualCredits(): ?int { return $this->actualCredits; }+ public function getBillingStatus(): ?string { return $this->billingStatus; }+ /** @return array<string, mixed> */+ public function getSummary(): array { return $this->summary; }+ public function getTargetResults(): mixed { return $this->targetResults; }+ public function getNotificationStatus(): mixed { return $this->notificationStatus; }+ public function getError(): ?string { return $this->error; }+ public function getCreatedAt(): ?string { return $this->createdAt; }+ public function getUpdatedAt(): ?string { return $this->updatedAt; }+}diff --git a/apps/php-sdk/src/Models/MonitorCheckDetail.php b/apps/php-sdk/src/Models/MonitorCheckDetail.phpnew file mode 100644index 000000000..f6709c81e--- /dev/null+++ b/apps/php-sdk/src/Models/MonitorCheckDetail.php@@ -0,0 +1,86 @@+<?php++declare(strict_types=1);++namespace Firecrawl\Models;++final class MonitorCheckDetail extends MonitorCheck+{+ /**+ * @param list<array<string, mixed>> $pages+ */+ public function __construct(+ ?string $id = null,+ ?string $monitorId = null,+ ?string $status = null,+ ?string $trigger = null,+ ?string $scheduledFor = null,+ ?string $startedAt = null,+ ?string $finishedAt = null,+ ?int $estimatedCredits = null,+ ?int $reservedCredits = null,+ ?int $actualCredits = null,+ ?string $billingStatus = null,+ array $summary = [],+ mixed $targetResults = null,+ mixed $notificationStatus = null,+ ?string $error = null,+ ?string $createdAt = null,+ ?string $updatedAt = null,+ private readonly array $pages = [],+ private readonly ?string $next = null,+ ) {+ parent::__construct(+ id: $id,+ monitorId: $monitorId,+ status: $status,+ trigger: $trigger,+ scheduledFor: $scheduledFor,+ startedAt: $startedAt,+ finishedAt: $finishedAt,+ estimatedCredits: $estimatedCredits,+ reservedCredits: $reservedCredits,+ actualCredits: $actualCredits,+ billingStatus: $billingStatus,+ summary: $summary,+ targetResults: $targetResults,+ notificationStatus: $notificationStatus,+ error: $error,+ createdAt: $createdAt,+ updatedAt: $updatedAt,+ );+ }++ /** @param array<string, mixed> $data */+ public static function fromArray(array $data): static+ {+ /** @var self $check */+ $check = new self(+ id: isset($data['id']) ? (string) $data['id'] : null,+ monitorId: isset($data['monitorId']) ? (string) $data['monitorId'] : null,+ status: isset($data['status']) ? (string) $data['status'] : null,+ trigger: isset($data['trigger']) ? (string) $data['trigger'] : null,+ scheduledFor: isset($data['scheduledFor']) ? (string) $data['scheduledFor'] : null,+ startedAt: isset($data['startedAt']) ? (string) $data['startedAt'] : null,+ finishedAt: isset($data['finishedAt']) ? (string) $data['finishedAt'] : null,+ estimatedCredits: isset($data['estimatedCredits']) ? (int) $data['estimatedCredits'] : null,+ reservedCredits: isset($data['reservedCredits']) ? (int) $data['reservedCredits'] : null,+ actualCredits: isset($data['actualCredits']) ? (int) $data['actualCredits'] : null,+ billingStatus: isset($data['billingStatus']) ? (string) $data['billingStatus'] : null,+ summary: isset($data['summary']) && is_array($data['summary']) ? $data['summary'] : [],+ targetResults: $data['targetResults'] ?? null,+ notificationStatus: $data['notificationStatus'] ?? null,+ error: isset($data['error']) ? (string) $data['error'] : null,+ createdAt: isset($data['createdAt']) ? (string) $data['createdAt'] : null,+ updatedAt: isset($data['updatedAt']) ? (string) $data['updatedAt'] : null,+ pages: isset($data['pages']) && is_array($data['pages']) ? $data['pages'] : [],+ next: isset($data['next']) ? (string) $data['next'] : null,+ );++ return $check;+ }++ /** @return list<array<string, mixed>> */+ public function getPages(): array { return $this->pages; }+ public function getNext(): ?string { return $this->next; }+}diff --git a/apps/php-sdk/src/Version.php b/apps/php-sdk/src/Version.phpindex e7da56ddc..8fb71f9b4 100644--- a/apps/php-sdk/src/Version.php+++ b/apps/php-sdk/src/Version.php@@ -6,5 +6,5 @@ namespace Firecrawl;final class Version{- public const SDK_VERSION = '1.1.1';+ public const SDK_VERSION = '1.2.0';}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex e42f93db3..8abe1b757 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.24.1"+__version__ = "4.25.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex 1c21a0a68..00fde14e0 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -92,6 +92,14 @@ class V2Proxy:self.get_batch_scrape_errors = client_instance.get_batch_scrape_errorsself.map = client_instance.map+ self.create_monitor = client_instance.create_monitor+ self.list_monitors = client_instance.list_monitors+ self.get_monitor = client_instance.get_monitor+ self.update_monitor = client_instance.update_monitor+ self.delete_monitor = client_instance.delete_monitor+ self.run_monitor = client_instance.run_monitor+ self.list_monitor_checks = client_instance.list_monitor_checks+ self.get_monitor_check = client_instance.get_monitor_checkself.get_concurrency = client_instance.get_concurrencyself.get_credit_usage = client_instance.get_credit_usageself.get_token_usage = client_instance.get_token_usage@@ -172,6 +180,14 @@ class AsyncV2Proxy:self.get_batch_scrape_errors = client_instance.get_batch_scrape_errorsself.map = client_instance.map+ self.create_monitor = client_instance.create_monitor+ self.list_monitors = client_instance.list_monitors+ self.get_monitor = client_instance.get_monitor+ self.update_monitor = client_instance.update_monitor+ self.delete_monitor = client_instance.delete_monitor+ self.run_monitor = client_instance.run_monitor+ self.list_monitor_checks = client_instance.list_monitor_checks+ self.get_monitor_check = client_instance.get_monitor_checkself.get_concurrency = client_instance.get_concurrencyself.get_credit_usage = client_instance.get_credit_usageself.get_token_usage = client_instance.get_token_usage@@ -242,6 +258,14 @@ class Firecrawl:self.parse = self._v2_client.parseself.search = self._v2_client.searchself.map = self._v2_client.map+ self.create_monitor = self._v2_client.create_monitor+ self.list_monitors = self._v2_client.list_monitors+ self.get_monitor = self._v2_client.get_monitor+ self.update_monitor = self._v2_client.update_monitor+ self.delete_monitor = self._v2_client.delete_monitor+ self.run_monitor = self._v2_client.run_monitor+ self.list_monitor_checks = self._v2_client.list_monitor_checks+ self.get_monitor_check = self._v2_client.get_monitor_checkself.crawl = self._v2_client.crawlself.start_crawl = self._v2_client.start_crawl@@ -336,6 +360,14 @@ class AsyncFirecrawl:self.parse = self._v2_client.parseself.search = self._v2_client.searchself.map = self._v2_client.map+ self.create_monitor = self._v2_client.create_monitor+ self.list_monitors = self._v2_client.list_monitors+ self.get_monitor = self._v2_client.get_monitor+ self.update_monitor = self._v2_client.update_monitor+ self.delete_monitor = self._v2_client.delete_monitor+ self.run_monitor = self._v2_client.run_monitor+ self.list_monitor_checks = self._v2_client.list_monitor_checks+ self.get_monitor_check = self._v2_client.get_monitor_checkself.start_crawl = self._v2_client.start_crawlself.get_crawl_status = self._v2_client.get_crawl_statusdiff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 88539b542..17084ccef 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -41,6 +41,14 @@ from .types import (Location,PaginationConfig,AgentOptions,+ Monitor,+ MonitorCheck,+ MonitorCheckDetail,+ MonitorCreateRequest,+ MonitorNotification,+ MonitorSchedule,+ MonitorTarget,+ MonitorUpdateRequest,)from .utils.http_client import HttpClientfrom .utils.error_handler import FirecrawlError@@ -55,6 +63,7 @@ from .methods import usage as usage_methodsfrom .methods import extract as extract_modulefrom .methods import agent as agent_modulefrom .methods import browser as browser_module+from .methods import monitor as monitor_modulefrom .watcher import Watcherclass FirecrawlClient:@@ -655,6 +664,112 @@ class FirecrawlClient:) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location]) else Nonereturn map_module.map(self.http_client, url, options)++ def create_monitor(+ self,+ name: str,+ schedule: Union[MonitorSchedule, Dict[str, Any]],+ targets: List[Union[MonitorTarget, Dict[str, Any]]],+ *,+ webhook: Optional[WebhookConfig] = None,+ notification: Optional[MonitorNotification] = None,+ retention_days: Optional[int] = None,+ ) -> Monitor:+ """Create a scheduled monitor."""+ if isinstance(schedule, dict):+ schedule = MonitorSchedule(**schedule)+ request = MonitorCreateRequest(+ name=name,+ schedule=schedule,+ targets=targets,+ webhook=webhook,+ notification=notification,+ retention_days=retention_days,+ )+ return monitor_module.create_monitor(self.http_client, request)++ def list_monitors(+ self,+ *,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+ ) -> List[Monitor]:+ """List monitors for the authenticated team."""+ return monitor_module.list_monitors(self.http_client, limit=limit, offset=offset)++ def get_monitor(self, monitor_id: str) -> Monitor:+ """Get a monitor by ID."""+ return monitor_module.get_monitor(self.http_client, monitor_id)++ def update_monitor(+ self,+ monitor_id: str,+ *,+ name: Optional[str] = None,+ status: Optional[Literal["active", "paused"]] = None,+ schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,+ webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,+ notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,+ targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,+ retention_days: Optional[int] = None,+ ) -> Monitor:+ """Update a monitor."""+ if isinstance(schedule, dict):+ schedule = MonitorSchedule(**schedule)+ request = MonitorUpdateRequest(+ name=name,+ status=status,+ schedule=schedule,+ webhook=webhook,+ notification=notification,+ targets=targets,+ retention_days=retention_days,+ )+ return monitor_module.update_monitor(self.http_client, monitor_id, request)++ def delete_monitor(self, monitor_id: str) -> bool:+ """Delete a monitor."""+ return monitor_module.delete_monitor(self.http_client, monitor_id)++ def run_monitor(self, monitor_id: str) -> MonitorCheck:+ """Run a monitor manually."""+ return monitor_module.run_monitor(self.http_client, monitor_id)++ def list_monitor_checks(+ self,+ monitor_id: str,+ *,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+ ) -> List[MonitorCheck]:+ """List checks for a monitor."""+ return monitor_module.list_monitor_checks(+ self.http_client,+ monitor_id,+ limit=limit,+ offset=offset,+ )++ def get_monitor_check(+ self,+ monitor_id: str,+ check_id: str,+ *,+ limit: Optional[int] = None,+ skip: Optional[int] = None,+ status: Optional[Literal["same", "new", "changed", "removed", "error"]] = None,+ pagination_config: Optional[PaginationConfig] = None,+ ) -> MonitorCheckDetail:+ """Get a monitor check with page results, auto-paginated by default."""+ return monitor_module.get_monitor_check(+ self.http_client,+ monitor_id,+ check_id,+ limit=limit,+ skip=skip,+ status=status,+ pagination_config=pagination_config,+ )def cancel_crawl(self, crawl_id: str) -> bool:"""diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex b41da8eff..834ac4407 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -36,6 +36,14 @@ from .types import (PDFAction,Location,PaginationConfig,+ Monitor,+ MonitorCheck,+ MonitorCheckDetail,+ MonitorCreateRequest,+ MonitorNotification,+ MonitorSchedule,+ MonitorTarget,+ MonitorUpdateRequest,)from .utils.http_client import HttpClientfrom .utils.http_client_async import AsyncHttpClient@@ -50,6 +58,7 @@ from .methods.aio import usage as async_usage # type: ignore[attr-defined]from .methods.aio import extract as async_extract # type: ignore[attr-defined]from .methods.aio import agent as async_agent # type: ignore[attr-defined]from .methods.aio import browser as async_browser # type: ignore[attr-defined]+from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]from .watcher_async import AsyncWatcher@@ -332,6 +341,112 @@ class AsyncFirecrawlClient:) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout]) else Nonereturn await async_map.map(self.async_http_client, url, options)+ async def create_monitor(+ self,+ name: str,+ schedule: Union[MonitorSchedule, Dict[str, Any]],+ targets: List[Union[MonitorTarget, Dict[str, Any]]],+ *,+ webhook: Optional[WebhookConfig] = None,+ notification: Optional[MonitorNotification] = None,+ retention_days: Optional[int] = None,+ ) -> Monitor:+ if isinstance(schedule, dict):+ schedule = MonitorSchedule(**schedule)+ request = MonitorCreateRequest(+ name=name,+ schedule=schedule,+ targets=targets,+ webhook=webhook,+ notification=notification,+ retention_days=retention_days,+ )+ return await async_monitor.create_monitor(self.async_http_client, request)++ async def list_monitors(+ self,+ *,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+ ) -> List[Monitor]:+ return await async_monitor.list_monitors(+ self.async_http_client,+ limit=limit,+ offset=offset,+ )++ async def get_monitor(self, monitor_id: str) -> Monitor:+ return await async_monitor.get_monitor(self.async_http_client, monitor_id)++ async def update_monitor(+ self,+ monitor_id: str,+ *,+ name: Optional[str] = None,+ status: Optional[Literal["active", "paused"]] = None,+ schedule: Optional[Union[MonitorSchedule, Dict[str, Any]]] = None,+ webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None,+ notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None,+ targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None,+ retention_days: Optional[int] = None,+ ) -> Monitor:+ if isinstance(schedule, dict):+ schedule = MonitorSchedule(**schedule)+ request = MonitorUpdateRequest(+ name=name,+ status=status,+ schedule=schedule,+ webhook=webhook,+ notification=notification,+ targets=targets,+ retention_days=retention_days,+ )+ return await async_monitor.update_monitor(+ self.async_http_client,+ monitor_id,+ request,+ )++ async def delete_monitor(self, monitor_id: str) -> bool:+ return await async_monitor.delete_monitor(self.async_http_client, monitor_id)++ async def run_monitor(self, monitor_id: str) -> MonitorCheck:+ return await async_monitor.run_monitor(self.async_http_client, monitor_id)++ async def list_monitor_checks(+ self,+ monitor_id: str,+ *,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+ ) -> List[MonitorCheck]:+ return await async_monitor.list_monitor_checks(+ self.async_http_client,+ monitor_id,+ limit=limit,+ offset=offset,+ )++ async def get_monitor_check(+ self,+ monitor_id: str,+ check_id: str,+ *,+ limit: Optional[int] = None,+ skip: Optional[int] = None,+ status: Optional[Literal["same", "new", "changed", "removed", "error"]] = None,+ pagination_config: Optional[PaginationConfig] = None,+ ) -> MonitorCheckDetail:+ return await async_monitor.get_monitor_check(+ self.async_http_client,+ monitor_id,+ check_id,+ limit=limit,+ skip=skip,+ status=status,+ pagination_config=pagination_config,+ )+async def start_batch_scrape(self, urls: List[str], **kwargs) -> Any:return await async_batch.start_batch_scrape(self.async_http_client, urls, **kwargs)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/monitor.py b/apps/python-sdk/firecrawl/v2/methods/aio/monitor.pynew file mode 100644index 000000000..fe98ffa05--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/aio/monitor.py@@ -0,0 +1,215 @@+import time+from typing import Any, Dict, List, Optional+from pydantic import BaseModel++from ...types import (+ Monitor,+ MonitorCheck,+ MonitorCheckDetail,+ MonitorCheckPage,+ MonitorCreateRequest,+ PaginationConfig,+ MonitorTarget,+ MonitorUpdateRequest,+ ScrapeOptions,+)+from ...utils.error_handler import handle_response_error+from ...utils.http_client_async import AsyncHttpClient+from ...utils.validation import prepare_scrape_options+++def _dump(value: Any) -> Any:+ if isinstance(value, ScrapeOptions):+ return prepare_scrape_options(value)+ if isinstance(value, MonitorTarget):+ data = value.model_dump(exclude_none=True, by_alias=True)+ if isinstance(value.scrape_options, ScrapeOptions):+ data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)+ return _prepare_target(data)+ if isinstance(value, BaseModel):+ return value.model_dump(exclude_none=True, by_alias=True)+ if isinstance(value, list):+ return [_dump(item) for item in value]+ if isinstance(value, dict):+ return {key: _dump(item) for key, item in value.items() if item is not None}+ return value+++def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:+ prepared = dict(target)+ if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):+ prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])+ if "crawlOptions" in prepared:+ prepared["crawlOptions"] = _dump(prepared["crawlOptions"])+ return prepared+++def _prepare_payload(request: Any) -> Dict[str, Any]:+ payload = _dump(request)+ if not isinstance(payload, dict):+ raise ValueError("Monitor request must be an object")+ if "targets" in payload:+ payload["targets"] = [+ _prepare_target(_dump(target))+ for target in payload.get("targets", [])+ ]+ return payload+++async def _data_or_error(response, action: str) -> Any:+ if response.status_code >= 400:+ handle_response_error(response, action)+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))+ return body.get("data")+++async def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:+ if response.status_code >= 400:+ handle_response_error(response, action)+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))+ data = body.get("data") or {}+ if body.get("next") is not None:+ data["next"] = body.get("next")+ return data+++async def _fetch_all_monitor_check_pages(+ client: AsyncHttpClient,+ next_url: str,+ initial_pages: List[MonitorCheckPage],+ pagination_config: Optional[PaginationConfig] = None,+) -> List[MonitorCheckPage]:+ pages = initial_pages.copy()+ current_url = next_url+ page_count = 0+ max_pages = pagination_config.max_pages if pagination_config else None+ max_results = pagination_config.max_results if pagination_config else None+ max_wait_time = pagination_config.max_wait_time if pagination_config else None+ start_time = time.monotonic()++ while current_url:+ if max_pages is not None and page_count >= max_pages:+ break+ if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:+ break++ response = await client.get(current_url)+ if response.status_code >= 400:+ break+ try:+ data = await _monitor_check_data_or_error(response, "get monitor check page")+ except Exception:+ break++ for page in data.get("pages") or []:+ if max_results is not None and len(pages) >= max_results:+ break+ pages.append(MonitorCheckPage(**page))++ if max_results is not None and len(pages) >= max_results:+ break++ current_url = data.get("next")+ page_count += 1++ return pages+++async def create_monitor(client: AsyncHttpClient, request: MonitorCreateRequest) -> Monitor:+ data = await _data_or_error(await client.post("/v2/monitor", _prepare_payload(request)), "create monitor")+ return Monitor(**data)+++async def list_monitors(client: AsyncHttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:+ params = []+ if limit is not None:+ params.append(f"limit={limit}")+ if offset is not None:+ params.append(f"offset={offset}")+ suffix = f"?{'&'.join(params)}" if params else ""+ data = await _data_or_error(await client.get(f"/v2/monitor{suffix}"), "list monitors")+ return [Monitor(**item) for item in data or []]+++async def get_monitor(client: AsyncHttpClient, monitor_id: str) -> Monitor:+ data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}"), "get monitor")+ return Monitor(**data)+++async def update_monitor(client: AsyncHttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:+ data = await _data_or_error(await client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")+ return Monitor(**data)+++async def delete_monitor(client: AsyncHttpClient, monitor_id: str) -> bool:+ response = await client.delete(f"/v2/monitor/{monitor_id}")+ if response.status_code >= 400:+ handle_response_error(response, "delete monitor")+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))+ return True+++async def run_monitor(client: AsyncHttpClient, monitor_id: str) -> MonitorCheck:+ data = await _data_or_error(await client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")+ return MonitorCheck(**data)+++async def list_monitor_checks(+ client: AsyncHttpClient,+ monitor_id: str,+ *,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+) -> List[MonitorCheck]:+ params = []+ if limit is not None:+ params.append(f"limit={limit}")+ if offset is not None:+ params.append(f"offset={offset}")+ suffix = f"?{'&'.join(params)}" if params else ""+ data = await _data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")+ return [MonitorCheck(**item) for item in data or []]+++async def get_monitor_check(+ client: AsyncHttpClient,+ monitor_id: str,+ check_id: str,+ *,+ limit: Optional[int] = None,+ skip: Optional[int] = None,+ status: Optional[str] = None,+ pagination_config: Optional[PaginationConfig] = None,+) -> MonitorCheckDetail:+ params = []+ if limit is not None:+ params.append(f"limit={limit}")+ if skip is not None:+ params.append(f"skip={skip}")+ if status is not None:+ params.append(f"status={status}")+ suffix = f"?{'&'.join(params)}" if params else ""+ data = await _monitor_check_data_or_error(await client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")+ detail = MonitorCheckDetail(**data)++ auto_paginate = pagination_config.auto_paginate if pagination_config else True+ if auto_paginate and detail.next and not (+ pagination_config+ and pagination_config.max_results is not None+ and len(detail.pages) >= pagination_config.max_results+ ):+ detail.pages = await _fetch_all_monitor_check_pages(+ client,+ detail.next,+ detail.pages,+ pagination_config,+ )+ detail.next = None++ return detaildiff --git a/apps/python-sdk/firecrawl/v2/methods/monitor.py b/apps/python-sdk/firecrawl/v2/methods/monitor.pynew file mode 100644index 000000000..2164206b0--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/monitor.py@@ -0,0 +1,214 @@+import time+from typing import Any, Dict, List, Optional+from pydantic import BaseModel++from ..types import (+ Monitor,+ MonitorCheck,+ MonitorCheckDetail,+ MonitorCheckPage,+ MonitorCreateRequest,+ PaginationConfig,+ MonitorTarget,+ MonitorUpdateRequest,+ ScrapeOptions,+)+from ..utils import HttpClient, handle_response_error+from ..utils.validation import prepare_scrape_options+++def _dump(value: Any) -> Any:+ if isinstance(value, ScrapeOptions):+ return prepare_scrape_options(value)+ if isinstance(value, MonitorTarget):+ data = value.model_dump(exclude_none=True, by_alias=True)+ if isinstance(value.scrape_options, ScrapeOptions):+ data["scrapeOptions"] = prepare_scrape_options(value.scrape_options)+ return _prepare_target(data)+ if isinstance(value, BaseModel):+ return value.model_dump(exclude_none=True, by_alias=True)+ if isinstance(value, list):+ return [_dump(item) for item in value]+ if isinstance(value, dict):+ return {key: _dump(item) for key, item in value.items() if item is not None}+ return value+++def _prepare_target(target: Dict[str, Any]) -> Dict[str, Any]:+ prepared = dict(target)+ if "scrapeOptions" in prepared and isinstance(prepared["scrapeOptions"], ScrapeOptions):+ prepared["scrapeOptions"] = prepare_scrape_options(prepared["scrapeOptions"])+ if "crawlOptions" in prepared:+ prepared["crawlOptions"] = _dump(prepared["crawlOptions"])+ return prepared+++def _prepare_payload(request: Any) -> Dict[str, Any]:+ payload = _dump(request)+ if not isinstance(payload, dict):+ raise ValueError("Monitor request must be an object")+ if "targets" in payload:+ payload["targets"] = [+ _prepare_target(_dump(target))+ for target in payload.get("targets", [])+ ]+ return payload+++def _data_or_error(response, action: str) -> Any:+ if not response.ok:+ handle_response_error(response, action)+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))+ return body.get("data")+++def _monitor_check_data_or_error(response, action: str) -> Dict[str, Any]:+ if not response.ok:+ handle_response_error(response, action)+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))+ data = body.get("data") or {}+ if body.get("next") is not None:+ data["next"] = body.get("next")+ return data+++def _fetch_all_monitor_check_pages(+ client: HttpClient,+ next_url: str,+ initial_pages: List[MonitorCheckPage],+ pagination_config: Optional[PaginationConfig] = None,+) -> List[MonitorCheckPage]:+ pages = initial_pages.copy()+ current_url = next_url+ page_count = 0+ max_pages = pagination_config.max_pages if pagination_config else None+ max_results = pagination_config.max_results if pagination_config else None+ max_wait_time = pagination_config.max_wait_time if pagination_config else None+ start_time = time.monotonic()++ while current_url:+ if max_pages is not None and page_count >= max_pages:+ break+ if max_wait_time is not None and (time.monotonic() - start_time) > max_wait_time:+ break++ response = client.get(current_url)+ if not response.ok:+ break+ try:+ data = _monitor_check_data_or_error(response, "get monitor check page")+ except Exception:+ break++ for page in data.get("pages") or []:+ if max_results is not None and len(pages) >= max_results:+ break+ pages.append(MonitorCheckPage(**page))++ if max_results is not None and len(pages) >= max_results:+ break++ current_url = data.get("next")+ page_count += 1++ return pages+++def create_monitor(client: HttpClient, request: MonitorCreateRequest) -> Monitor:+ data = _data_or_error(client.post("/v2/monitor", _prepare_payload(request)), "create monitor")+ return Monitor(**data)+++def list_monitors(client: HttpClient, *, limit: Optional[int] = None, offset: Optional[int] = None) -> List[Monitor]:+ params = []+ if limit is not None:+ params.append(f"limit={limit}")+ if offset is not None:+ params.append(f"offset={offset}")+ suffix = f"?{'&'.join(params)}" if params else ""+ data = _data_or_error(client.get(f"/v2/monitor{suffix}"), "list monitors")+ return [Monitor(**item) for item in data or []]+++def get_monitor(client: HttpClient, monitor_id: str) -> Monitor:+ data = _data_or_error(client.get(f"/v2/monitor/{monitor_id}"), "get monitor")+ return Monitor(**data)+++def update_monitor(client: HttpClient, monitor_id: str, request: MonitorUpdateRequest) -> Monitor:+ data = _data_or_error(client.patch(f"/v2/monitor/{monitor_id}", _prepare_payload(request)), "update monitor")+ return Monitor(**data)+++def delete_monitor(client: HttpClient, monitor_id: str) -> bool:+ response = client.delete(f"/v2/monitor/{monitor_id}")+ if not response.ok:+ handle_response_error(response, "delete monitor")+ body = response.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error occurred"))+ return True+++def run_monitor(client: HttpClient, monitor_id: str) -> MonitorCheck:+ data = _data_or_error(client.post(f"/v2/monitor/{monitor_id}/run", {}), "run monitor")+ return MonitorCheck(**data)+++def list_monitor_checks(+ client: HttpClient,+ monitor_id: str,+ *,+ limit: Optional[int] = None,+ offset: Optional[int] = None,+) -> List[MonitorCheck]:+ params = []+ if limit is not None:+ params.append(f"limit={limit}")+ if offset is not None:+ params.append(f"offset={offset}")+ suffix = f"?{'&'.join(params)}" if params else ""+ data = _data_or_error(client.get(f"/v2/monitor/{monitor_id}/checks{suffix}"), "list monitor checks")+ return [MonitorCheck(**item) for item in data or []]+++def get_monitor_check(+ client: HttpClient,+ monitor_id: str,+ check_id: str,+ *,+ limit: Optional[int] = None,+ skip: Optional[int] = None,+ status: Optional[str] = None,+ pagination_config: Optional[PaginationConfig] = None,+) -> MonitorCheckDetail:+ params = []+ if limit is not None:+ params.append(f"limit={limit}")+ if skip is not None:+ params.append(f"skip={skip}")+ if status is not None:+ params.append(f"status={status}")+ suffix = f"?{'&'.join(params)}" if params else ""+ data = _monitor_check_data_or_error(client.get(f"/v2/monitor/{monitor_id}/checks/{check_id}{suffix}"), "get monitor check")+ detail = MonitorCheckDetail(**data)++ auto_paginate = pagination_config.auto_paginate if pagination_config else True+ if auto_paginate and detail.next and not (+ pagination_config+ and pagination_config.max_results is not None+ and len(detail.pages) >= pagination_config.max_results+ ):+ detail.pages = _fetch_all_monitor_check_pages(+ client,+ detail.next,+ detail.pages,+ pagination_config,+ )+ detail.next = None++ return detaildiff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 9b06f0341..a2fe2368f 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -797,6 +797,136 @@ class MapResponse(BaseResponse[MapData]):pass+# Monitor types+class MonitorSchedule(BaseModel):+ """Cron schedule for a monitor."""++ cron: str+ timezone: str = "UTC"+++class MonitorEmailNotification(BaseModel):+ enabled: bool = False+ recipients: List[str] = []+ include_diffs: bool = Field(default=False, alias="includeDiffs")++ model_config = {"populate_by_name": True}+++class MonitorNotification(BaseModel):+ email: Optional[MonitorEmailNotification] = None+++class MonitorTarget(BaseModel):+ """A scrape or crawl target stored on a monitor."""++ model_config = {"extra": "allow", "populate_by_name": True}++ id: Optional[str] = None+ type: Literal["scrape", "crawl"]+ urls: Optional[List[str]] = None+ url: Optional[str] = None+ scrape_options: Optional[Union[ScrapeOptions, Dict[str, Any]]] = Field(default=None, alias="scrapeOptions")+ crawl_options: Optional[Dict[str, Any]] = Field(default=None, alias="crawlOptions")+++class MonitorCreateRequest(BaseModel):+ model_config = {"populate_by_name": True}++ name: str+ schedule: MonitorSchedule+ webhook: Optional[WebhookConfig] = None+ notification: Optional[MonitorNotification] = None+ targets: List[Union[MonitorTarget, Dict[str, Any]]]+ retention_days: Optional[int] = Field(default=None, alias="retentionDays")+++class MonitorUpdateRequest(BaseModel):+ model_config = {"populate_by_name": True}++ name: Optional[str] = None+ status: Optional[Literal["active", "paused"]] = None+ schedule: Optional[MonitorSchedule] = None+ webhook: Optional[Union[WebhookConfig, Dict[str, Any]]] = None+ notification: Optional[Union[MonitorNotification, Dict[str, Any]]] = None+ targets: Optional[List[Union[MonitorTarget, Dict[str, Any]]]] = None+ retention_days: Optional[int] = Field(default=None, alias="retentionDays")+++class MonitorSummary(BaseModel):+ total_pages: int = Field(default=0, alias="totalPages")+ same: int = 0+ changed: int = 0+ new: int = 0+ removed: int = 0+ error: int = 0++ model_config = {"populate_by_name": True}+++class Monitor(BaseModel):+ model_config = {"populate_by_name": True, "extra": "allow"}++ id: str+ name: str+ status: Literal["active", "paused", "deleted"]+ schedule: MonitorSchedule+ next_run_at: Optional[str] = Field(default=None, alias="nextRunAt")+ last_run_at: Optional[str] = Field(default=None, alias="lastRunAt")+ current_check_id: Optional[str] = Field(default=None, alias="currentCheckId")+ targets: List[Dict[str, Any]]+ webhook: Optional[Dict[str, Any]] = None+ notification: Optional[Dict[str, Any]] = None+ retention_days: int = Field(alias="retentionDays")+ estimated_credits_per_month: Optional[int] = Field(default=None, alias="estimatedCreditsPerMonth")+ last_check_summary: Optional[MonitorSummary] = Field(default=None, alias="lastCheckSummary")+ created_at: str = Field(alias="createdAt")+ updated_at: str = Field(alias="updatedAt")+++class MonitorCheck(BaseModel):+ model_config = {"populate_by_name": True, "extra": "allow"}++ id: str+ monitor_id: str = Field(alias="monitorId")+ status: Literal["queued", "running", "completed", "failed", "partial", "skipped_overlap"]+ trigger: Literal["scheduled", "manual"]+ scheduled_for: Optional[str] = Field(default=None, alias="scheduledFor")+ started_at: Optional[str] = Field(default=None, alias="startedAt")+ finished_at: Optional[str] = Field(default=None, alias="finishedAt")+ estimated_credits: Optional[int] = Field(default=None, alias="estimatedCredits")+ reserved_credits: Optional[int] = Field(default=None, alias="reservedCredits")+ actual_credits: Optional[int] = Field(default=None, alias="actualCredits")+ billing_status: Literal["not_applicable", "reserved", "confirmed", "released", "failed"] = Field(alias="billingStatus")+ summary: MonitorSummary+ target_results: Optional[Any] = Field(default=None, alias="targetResults")+ notification_status: Optional[Any] = Field(default=None, alias="notificationStatus")+ error: Optional[str] = None+ created_at: str = Field(alias="createdAt")+ updated_at: str = Field(alias="updatedAt")+++class MonitorCheckPage(BaseModel):+ model_config = {"populate_by_name": True, "extra": "allow"}++ id: str+ target_id: str = Field(alias="targetId")+ url: str+ status: Literal["same", "new", "changed", "removed", "error"]+ previous_scrape_id: Optional[str] = Field(default=None, alias="previousScrapeId")+ current_scrape_id: Optional[str] = Field(default=None, alias="currentScrapeId")+ status_code: Optional[int] = Field(default=None, alias="statusCode")+ error: Optional[str] = None+ metadata: Optional[Any] = None+ diff: Optional[Any] = None+ created_at: str = Field(alias="createdAt")+++class MonitorCheckDetail(MonitorCheck):+ pages: List[MonitorCheckPage] = []+ next: Optional[str] = None++# Extract typesclass ExtractRequest(BaseModel):"""Request for extract operations."""diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.pyindex cc4168246..7a75ddbb3 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py@@ -263,4 +263,50 @@ class HttpClient:time.sleep(backoff_factor * (2 ** attempt))# This should never be reached due to the exception handling above- raise last_exception or Exception("Unexpected error in DELETE request")\ No newline at end of file+ raise last_exception or Exception("Unexpected error in DELETE request")++ def patch(+ self,+ endpoint: str,+ data: Dict[str, Any],+ headers: Optional[Dict[str, str]] = None,+ timeout: Optional[float] = None,+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,+ ) -> requests.Response:+ """Make a PATCH request with retry logic."""+ if headers is None:+ headers = self._prepare_headers()+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factor++ payload = dict(data)+ payload['origin'] = f'python-sdk@{version}'+ url = self._build_url(endpoint)++ last_exception = None+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):+ try:+ response = requests.patch(+ url,+ json=payload,+ headers=headers,+ timeout=timeout+ )+ if response.status_code == 502 and attempt < num_attempts - 1:+ time.sleep(backoff_factor * (2 ** attempt))+ continue+ return response+ except requests.RequestException as e:+ last_exception = e+ if attempt == num_attempts - 1:+ raise e+ time.sleep(backoff_factor * (2 ** attempt))++ raise last_exception or Exception("Unexpected error in PATCH request")\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.pyindex 9c8e7dec1..52d14623f 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py@@ -201,3 +201,45 @@ class AsyncHttpClient:await asyncio.sleep(backoff_factor * (2 ** attempt))raise last_exception or Exception("Unexpected error in DELETE request")++ async def patch(+ self,+ endpoint: str,+ data: Dict[str, Any],+ headers: Optional[Dict[str, str]] = None,+ timeout: Optional[float] = None,+ retries: Optional[int] = None,+ backoff_factor: Optional[float] = None,+ ) -> httpx.Response:+ if timeout is None:+ timeout = self.timeout+ if retries is None:+ retries = self.max_retries+ if backoff_factor is None:+ backoff_factor = self.backoff_factor++ payload = dict(data)+ payload["origin"] = f"python-sdk@{version}"++ last_exception = None+ num_attempts = max(1, retries)++ for attempt in range(num_attempts):+ try:+ response = await self._client.patch(+ endpoint,+ json=payload,+ headers={**self._headers(), **(headers or {})},+ timeout=timeout,+ )+ if response.status_code == 502 and attempt < num_attempts - 1:+ await asyncio.sleep(backoff_factor * (2 ** attempt))+ continue+ return response+ except httpx.HTTPError as e:+ last_exception = e+ if attempt == num_attempts - 1:+ raise e+ await asyncio.sleep(backoff_factor * (2 ** attempt))++ raise last_exception or Exception("Unexpected error in PATCH request")diff --git a/apps/ruby-sdk/lib/firecrawl.rb b/apps/ruby-sdk/lib/firecrawl.rbindex 79af43268..64cd6dd5a 100644--- a/apps/ruby-sdk/lib/firecrawl.rb+++ b/apps/ruby-sdk/lib/firecrawl.rb@@ -23,4 +23,5 @@ require_relative "firecrawl/models/agent_response"require_relative "firecrawl/models/agent_status_response"require_relative "firecrawl/models/concurrency_check"require_relative "firecrawl/models/credit_usage"+require_relative "firecrawl/models/monitor"require_relative "firecrawl/client"diff --git a/apps/ruby-sdk/lib/firecrawl/client.rb b/apps/ruby-sdk/lib/firecrawl/client.rbindex 635c1bbba..7eaff8325 100644--- a/apps/ruby-sdk/lib/firecrawl/client.rb+++ b/apps/ruby-sdk/lib/firecrawl/client.rb@@ -1,6 +1,7 @@# frozen_string_literal: truerequire "json"+require "uri"module Firecrawl# Client for the Firecrawl v2 API.@@ -280,6 +281,83 @@ module FirecrawlModels::MapData.new(data)end+ # ================================================================+ # MONITOR+ # ================================================================++ def create_monitor(name:, schedule:, targets:, webhook: nil, notification: nil, retention_days: nil)+ body = {+ "name" => name,+ "schedule" => schedule,+ "targets" => targets,+ "webhook" => webhook,+ "notification" => notification,+ "retentionDays" => retention_days,+ }.compact+ raw = @http.post("/v2/monitor", body)+ Models::Monitor.new(raw["data"] || raw)+ end++ def list_monitors(limit: nil, offset: nil)+ raw = @http.get("/v2/monitor#{query(limit: limit, offset: offset)}")+ (raw["data"] || []).map { |item| Models::Monitor.new(item) }+ end++ def get_monitor(monitor_id)+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?++ raw = @http.get("/v2/monitor/#{monitor_id}")+ Models::Monitor.new(raw["data"] || raw)+ end++ def update_monitor(monitor_id, **attrs)+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?++ body = {+ "name" => attrs[:name],+ "status" => attrs[:status],+ "schedule" => attrs[:schedule],+ "webhook" => attrs[:webhook],+ "notification" => attrs[:notification],+ "targets" => attrs[:targets],+ "retentionDays" => attrs[:retention_days],+ }.compact+ raw = @http.patch("/v2/monitor/#{monitor_id}", body)+ Models::Monitor.new(raw["data"] || raw)+ end++ def delete_monitor(monitor_id)+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?++ @http.delete("/v2/monitor/#{monitor_id}")["success"] == true+ end++ def run_monitor(monitor_id)+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?++ raw = @http.post("/v2/monitor/#{monitor_id}/run", {})+ Models::MonitorCheck.new(raw["data"] || raw)+ end++ def list_monitor_checks(monitor_id, limit: nil, offset: nil)+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?++ raw = @http.get("/v2/monitor/#{monitor_id}/checks#{query(limit: limit, offset: offset)}")+ (raw["data"] || []).map { |item| Models::MonitorCheck.new(item) }+ end++ def get_monitor_check(monitor_id, check_id, limit: nil, skip: nil, status: nil, auto_paginate: true)+ raise ArgumentError, "Monitor ID is required" if monitor_id.nil?+ raise ArgumentError, "Check ID is required" if check_id.nil?++ params = query(limit: limit, skip: skip, status: status)+ raw = @http.get("/v2/monitor/#{monitor_id}/checks/#{check_id}#{params}")+ data = raw["data"] || raw+ data["next"] = raw["next"] if raw["next"]+ check = Models::MonitorCheckDetail.new(data)+ auto_paginate ? paginate_monitor_check(check) : check+ end+# ================================================================# SEARCH# ================================================================@@ -378,6 +456,11 @@ module Firecrawlprivate+ def query(**params)+ compact = params.compact+ compact.empty? ? "" : "?#{URI.encode_www_form(compact)}"+ end+def poll_crawl(job_id, poll_interval, timeout)deadline = Time.now + timeoutwhile Time.now < deadline@@ -423,5 +506,20 @@ module Firecrawlendjobend++ def paginate_monitor_check(check)+ check.pages ||= []+ current = check+ while current.next_url && !current.next_url.empty?+ raw = @http.get_absolute(current.next_url)+ data = raw["data"] || raw+ data["next"] = raw["next"] if raw["next"]+ next_page = Models::MonitorCheckDetail.new(data)+ check.pages.concat(next_page.pages) unless next_page.pages.empty?+ current = next_page+ end+ check.next_url = nil+ check+ endendenddiff --git a/apps/ruby-sdk/lib/firecrawl/http_client.rb b/apps/ruby-sdk/lib/firecrawl/http_client.rbindex 248619e5d..4d1a819e0 100644--- a/apps/ruby-sdk/lib/firecrawl/http_client.rb+++ b/apps/ruby-sdk/lib/firecrawl/http_client.rb@@ -59,6 +59,16 @@ module Firecrawlexecute_with_retry(uri, request)end+ # Sends a PATCH request with JSON body.+ def patch(path, body)+ uri = URI("#{@base_url}#{path}")+ request = Net::HTTP::Patch.new(uri)+ request["Authorization"] = "Bearer #{@api_key}"+ request["Content-Type"] = "application/json"+ request.body = JSON.generate(body)+ execute_with_retry(uri, request)+ end+# Sends a POST request with a multipart/form-data body.## @param path [String] API pathdiff --git a/apps/ruby-sdk/lib/firecrawl/models/monitor.rb b/apps/ruby-sdk/lib/firecrawl/models/monitor.rbnew file mode 100644index 000000000..8948baf23--- /dev/null+++ b/apps/ruby-sdk/lib/firecrawl/models/monitor.rb@@ -0,0 +1,68 @@+# frozen_string_literal: true++module Firecrawl+ module Models+ class Monitor+ attr_reader :id, :name, :status, :schedule, :next_run_at, :last_run_at,+ :current_check_id, :targets, :webhook, :notification,+ :retention_days, :estimated_credits_per_month,+ :last_check_summary, :created_at, :updated_at++ def initialize(data)+ @id = data["id"]+ @name = data["name"]+ @status = data["status"]+ @schedule = data["schedule"]+ @next_run_at = data["nextRunAt"]+ @last_run_at = data["lastRunAt"]+ @current_check_id = data["currentCheckId"]+ @targets = data["targets"] || []+ @webhook = data["webhook"]+ @notification = data["notification"]+ @retention_days = data["retentionDays"]+ @estimated_credits_per_month = data["estimatedCreditsPerMonth"]+ @last_check_summary = data["lastCheckSummary"]+ @created_at = data["createdAt"]+ @updated_at = data["updatedAt"]+ end+ end++ class MonitorCheck+ attr_reader :id, :monitor_id, :status, :trigger, :scheduled_for,+ :started_at, :finished_at, :estimated_credits,+ :reserved_credits, :actual_credits, :billing_status,+ :summary, :target_results, :notification_status, :error,+ :created_at, :updated_at++ def initialize(data)+ @id = data["id"]+ @monitor_id = data["monitorId"]+ @status = data["status"]+ @trigger = data["trigger"]+ @scheduled_for = data["scheduledFor"]+ @started_at = data["startedAt"]+ @finished_at = data["finishedAt"]+ @estimated_credits = data["estimatedCredits"]+ @reserved_credits = data["reservedCredits"]+ @actual_credits = data["actualCredits"]+ @billing_status = data["billingStatus"]+ @summary = data["summary"] || {}+ @target_results = data["targetResults"]+ @notification_status = data["notificationStatus"]+ @error = data["error"]+ @created_at = data["createdAt"]+ @updated_at = data["updatedAt"]+ end+ end++ class MonitorCheckDetail < MonitorCheck+ attr_accessor :pages, :next_url++ def initialize(data)+ super+ @pages = data["pages"] || []+ @next_url = data["next"]+ end+ end+ end+enddiff --git a/apps/ruby-sdk/lib/firecrawl/version.rb b/apps/ruby-sdk/lib/firecrawl/version.rbindex 33b639d21..02eba3e87 100644--- a/apps/ruby-sdk/lib/firecrawl/version.rb+++ b/apps/ruby-sdk/lib/firecrawl/version.rb@@ -1,5 +1,5 @@# frozen_string_literal: truemodule Firecrawl- VERSION = "1.3.1"+ VERSION = "1.4.0"enddiff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex 13078e92d..a0f539d34 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -250,7 +250,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "2.3.1"+version = "2.4.0"dependencies = ["mockito","reqwest",diff --git a/apps/rust-sdk/Cargo.toml b/apps/rust-sdk/Cargo.tomlindex 6de8905e7..509e1dcdc 100644--- a/apps/rust-sdk/Cargo.toml+++ b/apps/rust-sdk/Cargo.toml@@ -1,6 +1,6 @@[package]name = "firecrawl"-version = "2.3.1"+version = "2.4.0"edition = "2021"license = "MIT"homepage = "https://www.firecrawl.dev/"diff --git a/apps/rust-sdk/src/lib.rs b/apps/rust-sdk/src/lib.rsindex 379ecdbc9..4639f0cdb 100644--- a/apps/rust-sdk/src/lib.rs+++ b/apps/rust-sdk/src/lib.rs@@ -25,6 +25,7 @@ mod batch_scrape;mod client;mod crawl;mod map;+mod monitor;mod parse;mod scrape;mod search;@@ -36,6 +37,7 @@ pub use client::Client;pub use crawl::*;pub use error::FirecrawlError;pub use map::*;+pub use monitor::*;pub use parse::*;pub use scrape::*;pub use search::*;diff --git a/apps/rust-sdk/src/monitor.rs b/apps/rust-sdk/src/monitor.rsnew file mode 100644index 000000000..2b7e1b493--- /dev/null+++ b/apps/rust-sdk/src/monitor.rs@@ -0,0 +1,370 @@+//! Monitor endpoint for Firecrawl API v2.++use serde::{Deserialize, Serialize};+use serde_json::Value;++use crate::client::Client;+use crate::FirecrawlError;++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct MonitorSchedule {+ pub cron: String,+ pub timezone: Option<String>,+}++#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct CreateMonitorRequest {+ pub name: String,+ pub schedule: MonitorSchedule,+ pub targets: Vec<Value>,+ pub webhook: Option<Value>,+ pub notification: Option<Value>,+ pub retention_days: Option<u32>,+}++#[serde_with::skip_serializing_none]+#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct UpdateMonitorRequest {+ pub name: Option<String>,+ pub status: Option<String>,+ pub schedule: Option<MonitorSchedule>,+ pub targets: Option<Vec<Value>>,+ pub webhook: Option<Value>,+ pub notification: Option<Value>,+ pub retention_days: Option<u32>,+}++#[derive(Deserialize, Serialize, Debug, Default, Clone)]+#[serde(rename_all = "camelCase")]+pub struct MonitorSummary {+ pub total_pages: u32,+ pub same: u32,+ pub changed: u32,+ pub new: u32,+ pub removed: u32,+ pub error: u32,+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct Monitor {+ pub id: String,+ pub name: String,+ pub status: String,+ pub schedule: MonitorSchedule,+ pub next_run_at: Option<String>,+ pub last_run_at: Option<String>,+ pub current_check_id: Option<String>,+ pub targets: Vec<Value>,+ pub webhook: Option<Value>,+ pub notification: Option<Value>,+ pub retention_days: u32,+ pub estimated_credits_per_month: Option<u32>,+ pub last_check_summary: Option<MonitorSummary>,+ pub created_at: String,+ pub updated_at: String,+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct MonitorCheck {+ pub id: String,+ pub monitor_id: String,+ pub status: String,+ pub trigger: String,+ pub scheduled_for: Option<String>,+ pub started_at: Option<String>,+ pub finished_at: Option<String>,+ pub estimated_credits: Option<u32>,+ pub reserved_credits: Option<u32>,+ pub actual_credits: Option<u32>,+ pub billing_status: String,+ pub summary: MonitorSummary,+ pub target_results: Option<Value>,+ pub notification_status: Option<Value>,+ pub error: Option<String>,+ pub created_at: String,+ pub updated_at: String,+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct MonitorCheckPage {+ pub id: String,+ pub target_id: String,+ pub url: String,+ pub status: String,+ pub previous_scrape_id: Option<String>,+ pub current_scrape_id: Option<String>,+ pub status_code: Option<u16>,+ pub error: Option<String>,+ pub metadata: Option<Value>,+ pub diff: Option<Value>,+ pub created_at: String,+}++#[derive(Deserialize, Serialize, Debug, Clone)]+#[serde(rename_all = "camelCase")]+pub struct MonitorCheckDetail {+ #[serde(flatten)]+ pub check: MonitorCheck,+ pub pages: Vec<MonitorCheckPage>,+ pub next: Option<String>,+}++#[derive(Deserialize, Debug)]+#[serde(rename_all = "camelCase")]+struct DataResponse<T> {+ data: T,+}++#[derive(Deserialize, Debug)]+#[serde(rename_all = "camelCase")]+struct SuccessResponse {+ success: bool,+}++fn query(limit: Option<u32>, offset: Option<u32>, status: Option<&str>) -> String {+ let mut params = Vec::new();+ if let Some(limit) = limit {+ params.push(format!("limit={}", limit));+ }+ if let Some(offset) = offset {+ params.push(format!("offset={}", offset));+ }+ if let Some(status) = status {+ params.push(format!("status={}", status));+ }+ if params.is_empty() {+ String::new()+ } else {+ format!("?{}", params.join("&"))+ }+}++fn check_page_query(limit: Option<u32>, skip: Option<u32>, status: Option<&str>) -> String {+ let mut params = Vec::new();+ if let Some(limit) = limit {+ params.push(format!("limit={}", limit));+ }+ if let Some(skip) = skip {+ params.push(format!("skip={}", skip));+ }+ if let Some(status) = status {+ params.push(format!("status={}", status));+ }+ if params.is_empty() {+ String::new()+ } else {+ format!("?{}", params.join("&"))+ }+}++impl Client {+ pub async fn create_monitor(+ &self,+ request: CreateMonitorRequest,+ ) -> Result<Monitor, FirecrawlError> {+ let response = self+ .client+ .post(self.url("/monitor"))+ .headers(self.prepare_headers(None))+ .json(&request)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Creating monitor".to_string(), e))?;++ let response: DataResponse<Monitor> =+ self.handle_response(response, "create monitor").await?;+ Ok(response.data)+ }++ pub async fn list_monitors(+ &self,+ limit: Option<u32>,+ offset: Option<u32>,+ ) -> Result<Vec<Monitor>, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/monitor{}", query(limit, offset, None))))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Listing monitors".to_string(), e))?;++ let response: DataResponse<Vec<Monitor>> =+ self.handle_response(response, "list monitors").await?;+ Ok(response.data)+ }++ pub async fn get_monitor(+ &self,+ monitor_id: impl AsRef<str>,+ ) -> Result<Monitor, FirecrawlError> {+ let response = self+ .client+ .get(self.url(&format!("/monitor/{}", monitor_id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Getting monitor".to_string(), e))?;++ let response: DataResponse<Monitor> = self.handle_response(response, "get monitor").await?;+ Ok(response.data)+ }++ pub async fn update_monitor(+ &self,+ monitor_id: impl AsRef<str>,+ request: UpdateMonitorRequest,+ ) -> Result<Monitor, FirecrawlError> {+ let response = self+ .client+ .patch(self.url(&format!("/monitor/{}", monitor_id.as_ref())))+ .headers(self.prepare_headers(None))+ .json(&request)+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Updating monitor".to_string(), e))?;++ let response: DataResponse<Monitor> =+ self.handle_response(response, "update monitor").await?;+ Ok(response.data)+ }++ pub async fn delete_monitor(+ &self,+ monitor_id: impl AsRef<str>,+ ) -> Result<bool, FirecrawlError> {+ let response = self+ .client+ .delete(self.url(&format!("/monitor/{}", monitor_id.as_ref())))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Deleting monitor".to_string(), e))?;++ let response: SuccessResponse = self.handle_response(response, "delete monitor").await?;+ Ok(response.success)+ }++ pub async fn run_monitor(+ &self,+ monitor_id: impl AsRef<str>,+ ) -> Result<MonitorCheck, FirecrawlError> {+ let response = self+ .client+ .post(self.url(&format!("/monitor/{}/run", monitor_id.as_ref())))+ .headers(self.prepare_headers(None))+ .json(&serde_json::json!({}))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Running monitor".to_string(), e))?;++ let response: DataResponse<MonitorCheck> =+ self.handle_response(response, "run monitor").await?;+ Ok(response.data)+ }++ pub async fn list_monitor_checks(+ &self,+ monitor_id: impl AsRef<str>,+ limit: Option<u32>,+ offset: Option<u32>,+ ) -> Result<Vec<MonitorCheck>, FirecrawlError> {+ let path = format!(+ "/monitor/{}/checks{}",+ monitor_id.as_ref(),+ query(limit, offset, None)+ );+ let response = self+ .client+ .get(self.url(&path))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Listing monitor checks".to_string(), e))?;++ let response: DataResponse<Vec<MonitorCheck>> = self+ .handle_response(response, "list monitor checks")+ .await?;+ Ok(response.data)+ }++ pub async fn get_monitor_check(+ &self,+ monitor_id: impl AsRef<str>,+ check_id: impl AsRef<str>,+ limit: Option<u32>,+ skip: Option<u32>,+ status: Option<&str>,+ ) -> Result<MonitorCheckDetail, FirecrawlError> {+ let path = format!(+ "/monitor/{}/checks/{}{}",+ monitor_id.as_ref(),+ check_id.as_ref(),+ check_page_query(limit, skip, status)+ );+ let response = self+ .client+ .get(self.url(&path))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Getting monitor check".to_string(), e))?;++ let response: DataResponse<MonitorCheckDetail> =+ self.handle_response(response, "get monitor check").await?;+ let mut check = response.data;++ while let Some(next) = check.next.clone() {+ let response = self+ .client+ .get(next)+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| {+ FirecrawlError::HttpError("Getting monitor check page".to_string(), e)+ })?;+ let response: DataResponse<MonitorCheckDetail> = self+ .handle_response(response, "get monitor check page")+ .await?;+ check.pages.extend(response.data.pages);+ check.next = response.data.next;+ }++ Ok(check)+ }++ pub async fn get_monitor_check_page(+ &self,+ monitor_id: impl AsRef<str>,+ check_id: impl AsRef<str>,+ limit: Option<u32>,+ skip: Option<u32>,+ status: Option<&str>,+ ) -> Result<MonitorCheckDetail, FirecrawlError> {+ let path = format!(+ "/monitor/{}/checks/{}{}",+ monitor_id.as_ref(),+ check_id.as_ref(),+ check_page_query(limit, skip, status)+ );+ let response = self+ .client+ .get(self.url(&path))+ .headers(self.prepare_headers(None))+ .send()+ .await+ .map_err(|e| FirecrawlError::HttpError("Getting monitor check".to_string(), e))?;++ let response: DataResponse<MonitorCheckDetail> =+ self.handle_response(response, "get monitor check").await?;+ Ok(response.data)+ }+}