Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(monitors): reserve credits for previously seen PDF pages (#4889)
Rakshith Ramprakash58d5eef59a78
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.tsindex 1bce82397..47251522f 100644--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts@@ -3,6 +3,7 @@ import {describeIf,ALLOW_TEST_SUITE_WEBSITE,TEST_SELF_HOST,+ TEST_SUITE_WEBSITE,} from "../lib";import {idmux,@@ -31,6 +32,62 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {});}, 10000);+ it(+ "reserves the known PDF page cost before a repeat monitor run",+ async () => {+ const create = await monitorCreateRaw(+ {+ name: "PDF reservation monitor",+ schedule: { cron: "0 * * * *", timezone: "UTC" },+ targets: [+ {+ type: "scrape",+ urls: [+ `${TEST_SUITE_WEBSITE}/example-long.pdf?testId=${crypto.randomUUID()}`,+ ],+ scrapeOptions: { formats: ["markdown"] },+ },+ ],+ notification: { email: { enabled: false } },+ },+ identity,+ );+ expect(create.statusCode).toBe(200);+ const monitorId = create.body.data.id;+ const runToCompletion = async () => {+ const run = await monitorRunRaw(monitorId, identity);+ expect(run.statusCode).toBe(200);+ let check: any;+ for (let i = 0; i < 90; i++) {+ const raw = await monitorCheckRaw(monitorId, run.body.id, identity);+ expect(raw.statusCode).toBe(200);+ check = raw.body.data;+ if (+ ["completed", "partial", "failed", "skipped_no_credits"].includes(+ check.status,+ )+ )+ break;+ await new Promise(resolve => setTimeout(resolve, 1000));+ }+ expect(check.status).toBe("completed");+ return check;+ };+ try {+ const first = await runToCompletion();+ expect(first.estimatedCredits).toBe(1);+ expect(first.actualCredits).toBeGreaterThan(1);+ const second = await runToCompletion();+ expect(second.estimatedCredits).toBe(first.actualCredits);+ expect(second.reservedCredits).toBe(first.actualCredits);+ expect(second.actualCredits).toBe(first.actualCredits);+ } finally {+ await monitorDeleteRaw(monitorId, identity);+ }+ },+ 3 * scrapeTimeout,+ );+it("creates, lists, gets, pauses, and deletes a monitor", async () => {const create = await monitorCreateRaw({diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.tsindex 2abca7d70..0fd2c8b43 100644--- a/apps/api/src/services/monitoring/runner.finalization.test.ts+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts@@ -169,6 +169,33 @@ describe("monitor check finalization ownership", () => {}) as any);});+ it("requests the persisted credit estimate and skips the check when its hold is denied", async () => {+ current.status = "queued";+ current.autumn_lock_id = null;+ current.estimated_credits = 1009;+ vi.mocked(autumnService.lockCredits).mockResolvedValue({+ status: "denied",+ });+ await processMonitorCheckJob({+ checkId: current.id,+ monitorId: monitor.id,+ teamId: monitor.team_id,+ });+ expect(autumnService.lockCredits).toHaveBeenCalledWith(+ expect.objectContaining({+ value: 1009,+ lockId: "monitor_check-1",+ }),+ );+ expect(current).toMatchObject({+ status: "skipped_no_credits",+ actual_credits: 0,+ billing_status: "not_applicable",+ });+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();+ expect(bill).not.toHaveBeenCalled();+ });+it("settles and schedules a completed check once when another batch retained its running snapshot", async () => {await reconcileRunningMonitorChecks();await reconcileRunningMonitorChecks();diff --git a/apps/api/src/services/monitoring/store.reservation.test.ts b/apps/api/src/services/monitoring/store.reservation.test.tsnew file mode 100644index 000000000..12af408bc--- /dev/null+++ b/apps/api/src/services/monitoring/store.reservation.test.ts@@ -0,0 +1,135 @@+import { PgDialect } from "drizzle-orm/pg-core";++const { select, replicaSelect, where, insert, values } = vi.hoisted(() => {+ const where = vi.fn();+ const select = vi.fn(() => ({ from: () => ({ where }) }));+ const replicaSelect = vi.fn(() => ({+ from: () => ({ where: async () => [] }),+ }));+ const values = vi.fn((row: Record<string, unknown>) => ({+ returning: async () => [row],+ }));+ const insert = vi.fn(() => ({ values }));+ return { select, replicaSelect, where, insert, values };+});++vi.mock("../../db/connection", () => ({+ db: { select, insert },+ dbRr: { select: replicaSelect },+}));+vi.mock("../../db/rpc", () => ({ monitoringClaimDueMonitors: vi.fn() }));++import { createMonitorCheck } from "./store";+import type { MonitorRow } from "./types";++const monitor = {+ id: "22222222-2222-4222-8222-222222222222",+ team_id: "11111111-1111-4111-8111-111111111111",+ targets: [+ {+ id: "pdf-target",+ type: "scrape",+ urls: ["https://example.com/report.pdf"],+ },+ ],+ goal: "Watch for changes",+ judge_enabled: true,+} as MonitorRow;++beforeEach(() => {+ vi.clearAllMocks();+ where.mockResolvedValue([]);+});++describe("monitor check credit reservation", () => {+ it("reserves primary PDF history even when the replica has not received it", async () => {+ where.mockResolvedValue([+ {+ target_id: "pdf-target",+ url: "https://example.com/report.pdf",+ metadata: { numPages: 100, creditsUsed: 1 },+ },+ ]);+ const check = await createMonitorCheck({ monitor, trigger: "scheduled" });+ expect(select).toHaveBeenCalledTimes(1);+ expect(replicaSelect).not.toHaveBeenCalled();+ expect(check.estimated_credits).toBe(101);+ expect(values).toHaveBeenCalledWith(+ expect.objectContaining({+ monitor_id: monitor.id,+ estimated_credits: 101,+ }),+ );++ // Successful, active pages from this monitor/team/current target only.+ const predicate = new PgDialect().sqlToQuery(where.mock.calls[0][0]);+ expect(predicate.sql).toContain('"monitor_pages"."monitor_id" =');+ expect(predicate.sql).toContain('"monitor_pages"."team_id" =');+ expect(predicate.sql).toContain('"monitor_pages"."is_removed" =');+ expect(predicate.sql).toContain('"monitor_pages"."last_status" in');+ expect(predicate.params).toEqual([+ monitor.id,+ monitor.team_id,+ "pdf-target",+ false,+ "new",+ "changed",+ "same",+ ]);+ });++ it("uses the ordinary estimate on a first run with no saved pages", async () => {+ const check = await createMonitorCheck({ monitor, trigger: "manual" });+ expect(select).toHaveBeenCalledTimes(1);+ expect(where).toHaveBeenCalledTimes(1);+ expect(replicaSelect).not.toHaveBeenCalled();+ expect(check.estimated_credits).toBe(2);+ });++ it("does not start a check with an understated estimate when history cannot be read", async () => {+ where.mockRejectedValue(new Error("database unavailable"));+ await expect(+ createMonitorCheck({ monitor, trigger: "scheduled" }),+ ).rejects.toThrow(+ "Failed to read previous monitor pages for credit reservation",+ );+ expect(insert).not.toHaveBeenCalled();+ });++ it("does not read PDF history when parsing is disabled", async () => {+ const unparsed = {+ ...monitor,+ targets: [+ {+ ...monitor.targets[0],+ scrapeOptions: { parsers: [] },+ },+ ],+ } as MonitorRow;+ expect(+ (await createMonitorCheck({ monitor: unparsed, trigger: "manual" }))+ .estimated_credits,+ ).toBe(2);+ expect(select).not.toHaveBeenCalled();+ });++ it("keeps search monitors on their flat estimate without reading PDF history", async () => {+ const search = {+ ...monitor,+ targets: [+ {+ id: "search-target",+ type: "search",+ queries: ["docs"],+ maxResults: 10,+ depth: "raw",+ },+ ],+ } as MonitorRow;+ expect(+ (await createMonitorCheck({ monitor: search, trigger: "manual" }))+ .estimated_credits,+ ).toBe(2);+ expect(select).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 900220194..fc4abf47a 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -12,6 +12,115 @@ import type { MonitorTarget } from "./types";import { config } from "../../config";describe("monitoring store credit helpers", () => {+ describe("reserving credits using previous PDF results", () => {+ const target: MonitorTarget = {+ id: "crawl-target",+ type: "crawl",+ url: "https://example.com",+ crawlOptions: { limit: 69 },+ scrapeOptions: {},+ };+ const page = (+ numPages: unknown,+ url = "https://example.com/document.pdf",+ ) => ({+ target_id: target.id,+ url,+ metadata: { numPages },+ });++ it("covers the 905 PDF pages from the parked 943-credit monitor run", () => {+ const pdfPageCounts = [+ 2, 103, 22, 328, 50, 2, 133, 84, 60, 1, 3, 3, 55, 4, 5, 2, 2, 3, 2, 2,+ 1, 1, 1, 2, 10, 1, 2, 2, 2, 1, 9, 2, 2, 3,+ ];+ // 69 URL credits + 69 judge allowance + 871 additional PDF pages.+ expect(estimateMonitorCreditsPerRun([target], true)).toBe(138);+ expect(+ estimateMonitorCreditsPerRun(+ [target],+ true,+ pdfPageCounts.map(n => page(n)),+ ),+ ).toBe(1009);+ });++ it("keeps the current option costs and allowance for undiscovered URLs", () => {+ const jsonTarget = {+ ...target,+ scrapeOptions: { formats: [{ type: "json" }] },+ } as MonitorTarget;+ expect(+ estimateMonitorCreditsPerRun([jsonTarget], true, [page(100)]),+ ).toBe(69 * 6 + 99);+ });++ it("ignores saved pages for removed targets and removed scrape URLs", () => {+ const scrape: MonitorTarget = {+ id: target.id,+ type: "scrape",+ urls: ["https://example.com/current.pdf"],+ scrapeOptions: {},+ };+ expect(+ estimateMonitorCreditsPerRun([scrape], false, [+ page(100),+ { ...page(200, scrape.urls[0]), target_id: "old-target" },+ page(10, scrape.urls[0]),+ ]),+ ).toBe(10);+ });++ it("respects the current PDF maxPages limit", () => {+ const capped = {+ ...target,+ scrapeOptions: { parsers: [{ type: "pdf", maxPages: 5 }] },+ } as MonitorTarget;+ expect(estimateMonitorCreditsPerRun([capped], false, [page(100)])).toBe(+ 73,+ );+ });++ it("does not reserve PDF costs when parsing is disabled", () => {+ const unparsed = {+ ...target,+ scrapeOptions: { parsers: [] },+ } as MonitorTarget;+ expect(estimateMonitorCreditsPerRun([unparsed], true, [page(100)])).toBe(+ 138,+ );+ });++ it("caps historical PDF URLs to a reduced crawl limit conservatively", () => {+ const smaller = {+ ...target,+ crawlOptions: { limit: 1 },+ } as MonitorTarget;+ expect(+ estimateMonitorCreditsPerRun([smaller], true, [page(10), page(100)]),+ ).toBe(101);+ });++ it("includes redaction costs for each additional PDF page", () => {+ const redacted = {+ ...target,+ scrapeOptions: { redactPII: true },+ } as MonitorTarget;+ expect(estimateMonitorCreditsPerRun([redacted], false, [page(10)])).toBe(+ 69 + 9 * 5,+ );+ });++ it.each([null, undefined, "100", NaN, Infinity, -1, 0, 1.5])(+ "preserves the baseline for invalid or missing historical page count %s",+ numPages => {+ expect(+ estimateMonitorCreditsPerRun([target], true, [page(numPages)]),+ ).toBe(138);+ },+ );+ });+it("estimates goal-enabled scrape monitors from scrape option costs", () => {const targets: MonitorTarget[] = [{diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 94ec5a054..4b4851f53 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -1,11 +1,21 @@import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";-import { and, asc, count, desc, eq, isNull, ne, sql } from "drizzle-orm";+import {+ and,+ asc,+ count,+ desc,+ eq,+ inArray,+ isNull,+ ne,+ sql,+} from "drizzle-orm";import { db, dbRr } from "../../db/connection";import * as schema from "../../db/schema";import { monitoringClaimDueMonitors } from "../../db/rpc";import { config } from "../../config";-import { shouldParsePDF } from "../../controllers/v2/types";+import { getPDFMaxPages, shouldParsePDF } from "../../controllers/v2/types";import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";import {getNextMonitorRunAt,@@ -236,6 +246,9 @@ function estimateTargetPageCount(target: MonitorTarget): number {export function estimateMonitorCreditsPerRun(targets: MonitorTarget[],judgeEnabled: boolean = false,+ previousPages: Array<+ Pick<MonitorPageRow, "target_id" | "url" | "metadata">+ > = [],): number {const baseCredits = targets.reduce((sum, target) => sum + estimateTargetBaseCredits(target, judgeEnabled),@@ -251,7 +264,45 @@ export function estimateMonitorCreditsPerRun(0,): 0;- return baseCredits + judgeCredits;+ // The base estimate counts URLs, but a PDF bills per document page. Keep+ // the allowance for new URLs and add the extra pages we already know about.+ const pdfCredits = targets.reduce((sum, target) => {+ if (+ target.type === "search" ||+ !shouldParsePDF(target.scrapeOptions?.parsers as any)+ ) {+ return sum;+ }+ const maxPages = getPDFMaxPages(target.scrapeOptions?.parsers as any);+ const urls = target.type === "scrape" ? new Set(target.urls) : null;+ const extraPages = previousPages+ .filter(+ page => page.target_id === target.id && (!urls || urls.has(page.url)),+ )+ .map(page => {+ const numPages = (page.metadata as MonitorCreditMetadata | null)+ ?.numPages;+ if (+ typeof numPages !== "number" ||+ !Number.isSafeInteger(numPages) ||+ numPages <= 1+ ) {+ return 0;+ }+ return Math.max(0, Math.min(numPages, maxPages ?? numPages) - 1);+ })+ .sort((a, b) => b - a)+ .slice(0, estimateTargetPageCount(target));+ const creditsPerExtraPage = 1 + (target.scrapeOptions?.redactPII ? 4 : 0);+ return (+ sum ++ extraPages.reduce(+ (total, pages) => total + pages * creditsPerExtraPage,+ 0,+ )+ );+ }, 0);+ return baseCredits + judgeCredits + pdfCredits;}export function calculateMonitorCheckActualCreditsFromPages(@@ -637,9 +688,45 @@ export async function createMonitorCheck(params: {scheduledFor?: string | null;status?: MonitorCheckRow["status"];}): Promise<MonitorCheckRow> {+ const pdfTargetIds = params.monitor.targets+ .filter(+ target =>+ target.type !== "search" &&+ shouldParsePDF(target.scrapeOptions?.parsers as any),+ )+ .map(target => target.id);+ const previousPages = pdfTargetIds.length+ ? await run(+ () =>+ // This decides how much balance is held: a lagging replica can hide+ // pages saved by the previous run and under-reserve known PDF costs.+ db+ .select({+ target_id: schema.monitor_pages.target_id,+ url: schema.monitor_pages.url,+ metadata: schema.monitor_pages.metadata,+ })+ .from(schema.monitor_pages)+ .where(+ and(+ eq(schema.monitor_pages.monitor_id, params.monitor.id),+ eq(schema.monitor_pages.team_id, params.monitor.team_id),+ inArray(schema.monitor_pages.target_id, pdfTargetIds),+ eq(schema.monitor_pages.is_removed, false),+ inArray(schema.monitor_pages.last_status, [+ "new",+ "changed",+ "same",+ ]),+ ),+ ),+ "Failed to read previous monitor pages for credit reservation",+ )+ : [];const estimated = estimateMonitorCreditsPerRun(params.monitor.targets,Boolean(params.monitor.judge_enabled) && Boolean(params.monitor.goal),+ previousPages,);const [data] = await run(() =>