Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(monitors): reserve credits for previously seen PDF pages (#4889)

Rakshith Ramprakash58d5eef59a78
diff --git a/apps/api/src/__tests__/snips/v2/monitor.test.ts b/apps/api/src/__tests__/snips/v2/monitor.test.ts
index 1bce82397..47251522f 100644
--- a/apps/api/src/__tests__/snips/v2/monitor.test.ts
+++ b/apps/api/src/__tests__/snips/v2/monitor.test.ts
@@ -3,6 +3,7 @@ import {
describeIf,
ALLOW_TEST_SUITE_WEBSITE,
TEST_SELF_HOST,
+ TEST_SUITE_WEBSITE,
} from "../lib";
import {
idmux,
@@ -31,6 +32,62 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE && !TEST_SELF_HOST)("/v2/monitor", () => {
});
}, 10000);
+ it(
+ "reserves the known PDF page cost before a repeat monitor run",
+ async () => {
+ const create = await monitorCreateRaw(
+ {
+ name: "PDF reservation monitor",
+ schedule: { cron: "0 * * * *", timezone: "UTC" },
+ targets: [
+ {
+ type: "scrape",
+ urls: [
+ `${TEST_SUITE_WEBSITE}/example-long.pdf?testId=${crypto.randomUUID()}`,
+ ],
+ scrapeOptions: { formats: ["markdown"] },
+ },
+ ],
+ notification: { email: { enabled: false } },
+ },
+ identity,
+ );
+ expect(create.statusCode).toBe(200);
+ const monitorId = create.body.data.id;
+ const runToCompletion = async () => {
+ const run = await monitorRunRaw(monitorId, identity);
+ expect(run.statusCode).toBe(200);
+ let check: any;
+ for (let i = 0; i < 90; i++) {
+ const raw = await monitorCheckRaw(monitorId, run.body.id, identity);
+ expect(raw.statusCode).toBe(200);
+ check = raw.body.data;
+ if (
+ ["completed", "partial", "failed", "skipped_no_credits"].includes(
+ check.status,
+ )
+ )
+ break;
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+ expect(check.status).toBe("completed");
+ return check;
+ };
+ try {
+ const first = await runToCompletion();
+ expect(first.estimatedCredits).toBe(1);
+ expect(first.actualCredits).toBeGreaterThan(1);
+ const second = await runToCompletion();
+ expect(second.estimatedCredits).toBe(first.actualCredits);
+ expect(second.reservedCredits).toBe(first.actualCredits);
+ expect(second.actualCredits).toBe(first.actualCredits);
+ } finally {
+ await monitorDeleteRaw(monitorId, identity);
+ }
+ },
+ 3 * scrapeTimeout,
+ );
+
it("creates, lists, gets, pauses, and deletes a monitor", async () => {
const create = await monitorCreateRaw(
{
diff --git a/apps/api/src/services/monitoring/runner.finalization.test.ts b/apps/api/src/services/monitoring/runner.finalization.test.ts
index 2abca7d70..0fd2c8b43 100644
--- a/apps/api/src/services/monitoring/runner.finalization.test.ts
+++ b/apps/api/src/services/monitoring/runner.finalization.test.ts
@@ -169,6 +169,33 @@ describe("monitor check finalization ownership", () => {
}) as any);
});
+ it("requests the persisted credit estimate and skips the check when its hold is denied", async () => {
+ current.status = "queued";
+ current.autumn_lock_id = null;
+ current.estimated_credits = 1009;
+ vi.mocked(autumnService.lockCredits).mockResolvedValue({
+ status: "denied",
+ });
+ await processMonitorCheckJob({
+ checkId: current.id,
+ monitorId: monitor.id,
+ teamId: monitor.team_id,
+ });
+ expect(autumnService.lockCredits).toHaveBeenCalledWith(
+ expect.objectContaining({
+ value: 1009,
+ lockId: "monitor_check-1",
+ }),
+ );
+ expect(current).toMatchObject({
+ status: "skipped_no_credits",
+ actual_credits: 0,
+ billing_status: "not_applicable",
+ });
+ expect(autumnService.finalizeCreditsLock).not.toHaveBeenCalled();
+ expect(bill).not.toHaveBeenCalled();
+ });
+
it("settles and schedules a completed check once when another batch retained its running snapshot", async () => {
await reconcileRunningMonitorChecks();
await reconcileRunningMonitorChecks();
diff --git a/apps/api/src/services/monitoring/store.reservation.test.ts b/apps/api/src/services/monitoring/store.reservation.test.ts
new file mode 100644
index 000000000..12af408bc
--- /dev/null
+++ b/apps/api/src/services/monitoring/store.reservation.test.ts
@@ -0,0 +1,135 @@
+import { PgDialect } from "drizzle-orm/pg-core";
+
+const { select, replicaSelect, where, insert, values } = vi.hoisted(() => {
+ const where = vi.fn();
+ const select = vi.fn(() => ({ from: () => ({ where }) }));
+ const replicaSelect = vi.fn(() => ({
+ from: () => ({ where: async () => [] }),
+ }));
+ const values = vi.fn((row: Record<string, unknown>) => ({
+ returning: async () => [row],
+ }));
+ const insert = vi.fn(() => ({ values }));
+ return { select, replicaSelect, where, insert, values };
+});
+
+vi.mock("../../db/connection", () => ({
+ db: { select, insert },
+ dbRr: { select: replicaSelect },
+}));
+vi.mock("../../db/rpc", () => ({ monitoringClaimDueMonitors: vi.fn() }));
+
+import { createMonitorCheck } from "./store";
+import type { MonitorRow } from "./types";
+
+const monitor = {
+ id: "22222222-2222-4222-8222-222222222222",
+ team_id: "11111111-1111-4111-8111-111111111111",
+ targets: [
+ {
+ id: "pdf-target",
+ type: "scrape",
+ urls: ["https://example.com/report.pdf"],
+ },
+ ],
+ goal: "Watch for changes",
+ judge_enabled: true,
+} as MonitorRow;
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ where.mockResolvedValue([]);
+});
+
+describe("monitor check credit reservation", () => {
+ it("reserves primary PDF history even when the replica has not received it", async () => {
+ where.mockResolvedValue([
+ {
+ target_id: "pdf-target",
+ url: "https://example.com/report.pdf",
+ metadata: { numPages: 100, creditsUsed: 1 },
+ },
+ ]);
+ const check = await createMonitorCheck({ monitor, trigger: "scheduled" });
+ expect(select).toHaveBeenCalledTimes(1);
+ expect(replicaSelect).not.toHaveBeenCalled();
+ expect(check.estimated_credits).toBe(101);
+ expect(values).toHaveBeenCalledWith(
+ expect.objectContaining({
+ monitor_id: monitor.id,
+ estimated_credits: 101,
+ }),
+ );
+
+ // Successful, active pages from this monitor/team/current target only.
+ const predicate = new PgDialect().sqlToQuery(where.mock.calls[0][0]);
+ expect(predicate.sql).toContain('"monitor_pages"."monitor_id" =');
+ expect(predicate.sql).toContain('"monitor_pages"."team_id" =');
+ expect(predicate.sql).toContain('"monitor_pages"."is_removed" =');
+ expect(predicate.sql).toContain('"monitor_pages"."last_status" in');
+ expect(predicate.params).toEqual([
+ monitor.id,
+ monitor.team_id,
+ "pdf-target",
+ false,
+ "new",
+ "changed",
+ "same",
+ ]);
+ });
+
+ it("uses the ordinary estimate on a first run with no saved pages", async () => {
+ const check = await createMonitorCheck({ monitor, trigger: "manual" });
+ expect(select).toHaveBeenCalledTimes(1);
+ expect(where).toHaveBeenCalledTimes(1);
+ expect(replicaSelect).not.toHaveBeenCalled();
+ expect(check.estimated_credits).toBe(2);
+ });
+
+ it("does not start a check with an understated estimate when history cannot be read", async () => {
+ where.mockRejectedValue(new Error("database unavailable"));
+ await expect(
+ createMonitorCheck({ monitor, trigger: "scheduled" }),
+ ).rejects.toThrow(
+ "Failed to read previous monitor pages for credit reservation",
+ );
+ expect(insert).not.toHaveBeenCalled();
+ });
+
+ it("does not read PDF history when parsing is disabled", async () => {
+ const unparsed = {
+ ...monitor,
+ targets: [
+ {
+ ...monitor.targets[0],
+ scrapeOptions: { parsers: [] },
+ },
+ ],
+ } as MonitorRow;
+ expect(
+ (await createMonitorCheck({ monitor: unparsed, trigger: "manual" }))
+ .estimated_credits,
+ ).toBe(2);
+ expect(select).not.toHaveBeenCalled();
+ });
+
+ it("keeps search monitors on their flat estimate without reading PDF history", async () => {
+ const search = {
+ ...monitor,
+ targets: [
+ {
+ id: "search-target",
+ type: "search",
+ queries: ["docs"],
+ maxResults: 10,
+ depth: "raw",
+ },
+ ],
+ } as MonitorRow;
+ expect(
+ (await createMonitorCheck({ monitor: search, trigger: "manual" }))
+ .estimated_credits,
+ ).toBe(2);
+ expect(select).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index 900220194..fc4abf47a 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -12,6 +12,115 @@ import type { MonitorTarget } from "./types";
import { config } from "../../config";
describe("monitoring store credit helpers", () => {
+ describe("reserving credits using previous PDF results", () => {
+ const target: MonitorTarget = {
+ id: "crawl-target",
+ type: "crawl",
+ url: "https://example.com",
+ crawlOptions: { limit: 69 },
+ scrapeOptions: {},
+ };
+ const page = (
+ numPages: unknown,
+ url = "https://example.com/document.pdf",
+ ) => ({
+ target_id: target.id,
+ url,
+ metadata: { numPages },
+ });
+
+ it("covers the 905 PDF pages from the parked 943-credit monitor run", () => {
+ const pdfPageCounts = [
+ 2, 103, 22, 328, 50, 2, 133, 84, 60, 1, 3, 3, 55, 4, 5, 2, 2, 3, 2, 2,
+ 1, 1, 1, 2, 10, 1, 2, 2, 2, 1, 9, 2, 2, 3,
+ ];
+ // 69 URL credits + 69 judge allowance + 871 additional PDF pages.
+ expect(estimateMonitorCreditsPerRun([target], true)).toBe(138);
+ expect(
+ estimateMonitorCreditsPerRun(
+ [target],
+ true,
+ pdfPageCounts.map(n => page(n)),
+ ),
+ ).toBe(1009);
+ });
+
+ it("keeps the current option costs and allowance for undiscovered URLs", () => {
+ const jsonTarget = {
+ ...target,
+ scrapeOptions: { formats: [{ type: "json" }] },
+ } as MonitorTarget;
+ expect(
+ estimateMonitorCreditsPerRun([jsonTarget], true, [page(100)]),
+ ).toBe(69 * 6 + 99);
+ });
+
+ it("ignores saved pages for removed targets and removed scrape URLs", () => {
+ const scrape: MonitorTarget = {
+ id: target.id,
+ type: "scrape",
+ urls: ["https://example.com/current.pdf"],
+ scrapeOptions: {},
+ };
+ expect(
+ estimateMonitorCreditsPerRun([scrape], false, [
+ page(100),
+ { ...page(200, scrape.urls[0]), target_id: "old-target" },
+ page(10, scrape.urls[0]),
+ ]),
+ ).toBe(10);
+ });
+
+ it("respects the current PDF maxPages limit", () => {
+ const capped = {
+ ...target,
+ scrapeOptions: { parsers: [{ type: "pdf", maxPages: 5 }] },
+ } as MonitorTarget;
+ expect(estimateMonitorCreditsPerRun([capped], false, [page(100)])).toBe(
+ 73,
+ );
+ });
+
+ it("does not reserve PDF costs when parsing is disabled", () => {
+ const unparsed = {
+ ...target,
+ scrapeOptions: { parsers: [] },
+ } as MonitorTarget;
+ expect(estimateMonitorCreditsPerRun([unparsed], true, [page(100)])).toBe(
+ 138,
+ );
+ });
+
+ it("caps historical PDF URLs to a reduced crawl limit conservatively", () => {
+ const smaller = {
+ ...target,
+ crawlOptions: { limit: 1 },
+ } as MonitorTarget;
+ expect(
+ estimateMonitorCreditsPerRun([smaller], true, [page(10), page(100)]),
+ ).toBe(101);
+ });
+
+ it("includes redaction costs for each additional PDF page", () => {
+ const redacted = {
+ ...target,
+ scrapeOptions: { redactPII: true },
+ } as MonitorTarget;
+ expect(estimateMonitorCreditsPerRun([redacted], false, [page(10)])).toBe(
+ 69 + 9 * 5,
+ );
+ });
+
+ it.each([null, undefined, "100", NaN, Infinity, -1, 0, 1.5])(
+ "preserves the baseline for invalid or missing historical page count %s",
+ numPages => {
+ expect(
+ estimateMonitorCreditsPerRun([target], true, [page(numPages)]),
+ ).toBe(138);
+ },
+ );
+ });
+
it("estimates goal-enabled scrape monitors from scrape option costs", () => {
const targets: MonitorTarget[] = [
{
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 94ec5a054..4b4851f53 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -1,11 +1,21 @@
import { createHash } from "crypto";
import { v7 as uuidv7 } from "uuid";
-import { and, asc, count, desc, eq, isNull, ne, sql } from "drizzle-orm";
+import {
+ and,
+ asc,
+ count,
+ desc,
+ eq,
+ inArray,
+ isNull,
+ ne,
+ sql,
+} from "drizzle-orm";
import { db, dbRr } from "../../db/connection";
import * as schema from "../../db/schema";
import { monitoringClaimDueMonitors } from "../../db/rpc";
import { config } from "../../config";
-import { shouldParsePDF } from "../../controllers/v2/types";
+import { getPDFMaxPages, shouldParsePDF } from "../../controllers/v2/types";
import { isXTwitterUrl } from "../../scraper/scrapeURL/engines/x-twitter/url";
import {
getNextMonitorRunAt,
@@ -236,6 +246,9 @@ function estimateTargetPageCount(target: MonitorTarget): number {
export function estimateMonitorCreditsPerRun(
targets: MonitorTarget[],
judgeEnabled: boolean = false,
+ previousPages: Array<
+ Pick<MonitorPageRow, "target_id" | "url" | "metadata">
+ > = [],
): number {
const baseCredits = targets.reduce(
(sum, target) => sum + estimateTargetBaseCredits(target, judgeEnabled),
@@ -251,7 +264,45 @@ export function estimateMonitorCreditsPerRun(
0,
)
: 0;
- return baseCredits + judgeCredits;
+ // The base estimate counts URLs, but a PDF bills per document page. Keep
+ // the allowance for new URLs and add the extra pages we already know about.
+ const pdfCredits = targets.reduce((sum, target) => {
+ if (
+ target.type === "search" ||
+ !shouldParsePDF(target.scrapeOptions?.parsers as any)
+ ) {
+ return sum;
+ }
+ const maxPages = getPDFMaxPages(target.scrapeOptions?.parsers as any);
+ const urls = target.type === "scrape" ? new Set(target.urls) : null;
+ const extraPages = previousPages
+ .filter(
+ page => page.target_id === target.id && (!urls || urls.has(page.url)),
+ )
+ .map(page => {
+ const numPages = (page.metadata as MonitorCreditMetadata | null)
+ ?.numPages;
+ if (
+ typeof numPages !== "number" ||
+ !Number.isSafeInteger(numPages) ||
+ numPages <= 1
+ ) {
+ return 0;
+ }
+ return Math.max(0, Math.min(numPages, maxPages ?? numPages) - 1);
+ })
+ .sort((a, b) => b - a)
+ .slice(0, estimateTargetPageCount(target));
+ const creditsPerExtraPage = 1 + (target.scrapeOptions?.redactPII ? 4 : 0);
+ return (
+ sum +
+ extraPages.reduce(
+ (total, pages) => total + pages * creditsPerExtraPage,
+ 0,
+ )
+ );
+ }, 0);
+ return baseCredits + judgeCredits + pdfCredits;
}
export function calculateMonitorCheckActualCreditsFromPages(
@@ -637,9 +688,45 @@ export async function createMonitorCheck(params: {
scheduledFor?: string | null;
status?: MonitorCheckRow["status"];
}): Promise<MonitorCheckRow> {
+ const pdfTargetIds = params.monitor.targets
+ .filter(
+ target =>
+ target.type !== "search" &&
+ shouldParsePDF(target.scrapeOptions?.parsers as any),
+ )
+ .map(target => target.id);
+ const previousPages = pdfTargetIds.length
+ ? await run(
+ () =>
+ // This decides how much balance is held: a lagging replica can hide
+ // pages saved by the previous run and under-reserve known PDF costs.
+ db
+ .select({
+ target_id: schema.monitor_pages.target_id,
+ url: schema.monitor_pages.url,
+ metadata: schema.monitor_pages.metadata,
+ })
+ .from(schema.monitor_pages)
+ .where(
+ and(
+ eq(schema.monitor_pages.monitor_id, params.monitor.id),
+ eq(schema.monitor_pages.team_id, params.monitor.team_id),
+ inArray(schema.monitor_pages.target_id, pdfTargetIds),
+ eq(schema.monitor_pages.is_removed, false),
+ inArray(schema.monitor_pages.last_status, [
+ "new",
+ "changed",
+ "same",
+ ]),
+ ),
+ ),
+ "Failed to read previous monitor pages for credit reservation",
+ )
+ : [];
const estimated = estimateMonitorCreditsPerRun(
params.monitor.targets,
Boolean(params.monitor.judge_enabled) && Boolean(params.monitor.goal),
+ previousPages,
);
const [data] = await run(
() =>