Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Refactor monitoring credit accounting and metadata

Developers Digest128a619aff3e
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
index aa8fcc120..6398948ea 100644
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@ -2,7 +2,6 @@ import { NuQJob } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
import { logger as _logger } from "../../lib/logger";
import { createWebhookSender, WebhookEvent } from "../webhook";
-import { billTeam } from "../billing/credit_billing";
import { computeAndPersistPageDiff } from "./diff-orchestrator";
import { derivePageIsMeaningful } from "./page-events";
import {
@@ -13,8 +12,6 @@ import {
upsertMonitorPage,
} from "./store";
-const JUDGE_CREDIT_COST = 1;
-
const logger = _logger.child({ module: "monitoring-results" });
function getDocumentUrl(doc: any, fallback: string): string {
@@ -174,6 +171,8 @@ export async function recordMonitorScrapeSuccess(
metadata: {
title: doc?.metadata?.title ?? null,
statusCode: getDocumentStatusCode(doc),
+ contentType: doc?.metadata?.contentType ?? null,
+ numPages: doc?.metadata?.numPages ?? null,
creditsUsed: doc?.metadata?.creditsUsed ?? null,
},
});
@@ -195,6 +194,8 @@ export async function recordMonitorScrapeSuccess(
status_code: getDocumentStatusCode(doc),
metadata: {
title: doc?.metadata?.title ?? null,
+ contentType: doc?.metadata?.contentType ?? null,
+ numPages: doc?.metadata?.numPages ?? null,
creditsUsed: doc?.metadata?.creditsUsed ?? null,
},
judgment: judgment ?? null,
@@ -213,25 +214,6 @@ export async function recordMonitorScrapeSuccess(
judgmentMeaningful: judgment?.meaningful,
});
- if (judgment) {
- try {
- await billTeam(
- job.data.team_id,
- undefined,
- JUDGE_CREDIT_COST,
- job.data.apiKeyId ?? null,
- { endpoint: "monitor", jobId: monitoring.checkId },
- logger,
- );
- } catch (error) {
- logger.warn("Failed to bill judge credit", {
- error,
- monitorId: monitoring.monitorId,
- checkId: monitoring.checkId,
- });
- }
- }
-
await sendMonitorPageWebhook({
teamId: job.data.team_id,
monitorId: monitoring.monitorId,
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index 217925c92..c728ea89a 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -2,9 +2,34 @@ jest.mock("uuid", () => ({
v7: () => "test-uuid-v7",
}));
-import { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS } from "./runner";
+import {
+ estimateActualCredits,
+ isMonitorCheckStale,
+ MONITOR_CHECK_STALE_TIMEOUT_MS,
+} from "./runner";
describe("monitoring runner", () => {
+ describe("estimateActualCredits", () => {
+ it("prefers scrape-reported credits when present", () => {
+ expect(estimateActualCredits({ metadata: { creditsUsed: 9 } }, {})).toBe(
+ 9,
+ );
+ });
+
+ it("bills parsed PDFs by extracted page count", () => {
+ expect(estimateActualCredits({ metadata: { numPages: 4 } }, {})).toBe(4);
+ });
+
+ it("adds PDF page credits on top of JSON scrape credits", () => {
+ expect(
+ estimateActualCredits(
+ { metadata: { numPages: 3 } },
+ { formats: ["json"] },
+ ),
+ ).toBe(7);
+ });
+ });
+
describe("isMonitorCheckStale", () => {
const now = new Date("2026-05-06T12:00:00.000Z");
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 25c1d94d1..fe2035491 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -23,11 +23,13 @@ import {
type ScrapeOptions,
crawlRequestSchema,
scrapeRequestSchema,
+ shouldParsePDF,
toV0CrawlerOptions,
} from "../../controllers/v2/types";
import { createWebhookSender, WebhookEvent } from "../webhook";
import { sendMonitoringEmailSummary } from "../notification/monitoring_email";
import {
+ calculateMonitorCheckActualCredits,
getMonitorCheck,
getMonitorForUpdate,
getMonitorPage,
@@ -64,6 +66,10 @@ export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };
const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;
const MONITOR_CHECK_PAGE_SCAN_LIMIT = 100_000;
+const BASE_SCRAPE_CREDITS_PER_DOCUMENT = 1;
+const JSON_SCRAPE_CREDITS_PER_DOCUMENT = 5;
+const SCRAPE_OPTION_CREDIT_BONUS = 4;
+const PDF_CREDITS_PER_PAGE = 1;
const TERMINAL_CHECK_STATUSES = new Set([
"completed",
"partial",
@@ -211,15 +217,63 @@ function getDocumentStatusCode(doc: any): number | null {
: null;
}
-function estimateActualCredits(doc: any, options: any): number {
+function hasScrapeFormat(formats: unknown[], type: string): boolean {
+ return formats.some(format =>
+ typeof format === "string"
+ ? format === type
+ : (format as any)?.type === type,
+ );
+}
+
+function hasJsonChangeTracking(format: unknown): boolean {
+ return (
+ typeof format === "object" &&
+ format !== null &&
+ (format as any).type === "changeTracking" &&
+ Array.isArray((format as any).modes) &&
+ (format as any).modes.includes("json")
+ );
+}
+
+export function estimateActualCredits(doc: any, options: any): number {
if (typeof doc?.metadata?.creditsUsed === "number") {
return doc.metadata.creditsUsed;
}
const formats = Array.isArray(options?.formats) ? options.formats : [];
- const hasJson = formats.some((format: any) =>
- typeof format === "string" ? format === "json" : format?.type === "json",
+ const hasJson =
+ hasScrapeFormat(formats, "json") || formats.some(hasJsonChangeTracking);
+ const baseCredits = hasJson
+ ? JSON_SCRAPE_CREDITS_PER_DOCUMENT
+ : BASE_SCRAPE_CREDITS_PER_DOCUMENT +
+ (options?.lockdown ? SCRAPE_OPTION_CREDIT_BONUS : 0);
+ const formatBonusCount = [
+ hasScrapeFormat(formats, "question") || hasScrapeFormat(formats, "query"),
+ hasScrapeFormat(formats, "highlights"),
+ hasScrapeFormat(formats, "audio"),
+ hasScrapeFormat(formats, "video"),
+ ].reduce((count, enabled) => count + (enabled ? 1 : 0), 0);
+ const pdfPages =
+ typeof doc?.metadata?.numPages === "number" &&
+ Number.isFinite(doc.metadata.numPages)
+ ? Math.max(1, Math.ceil(doc.metadata.numPages))
+ : 1;
+ const pdfCredits =
+ shouldParsePDF(options?.parsers) && pdfPages > 1
+ ? PDF_CREDITS_PER_PAGE * (pdfPages - 1)
+ : 0;
+ const proxyCredits =
+ doc?.metadata?.proxyUsed === "stealth" ||
+ options?.proxy === "stealth" ||
+ options?.proxy === "enhanced"
+ ? SCRAPE_OPTION_CREDIT_BONUS
+ : 0;
+
+ return (
+ baseCredits +
+ formatBonusCount * SCRAPE_OPTION_CREDIT_BONUS +
+ pdfCredits +
+ proxyCredits
);
- return hasJson ? 5 : 1;
}
async function runSingleScrape(params: {
@@ -290,6 +344,7 @@ async function diffAndPersistPage(params: {
scrapeId: string;
doc: any;
source: "explicit" | "discovered";
+ creditsUsed?: number;
}): Promise<PageResult> {
const previous = await getMonitorPage({
monitorId: params.monitor.id,
@@ -333,6 +388,9 @@ async function diffAndPersistPage(params: {
metadata: {
title: params.doc?.metadata?.title ?? null,
statusCode: getDocumentStatusCode(params.doc),
+ contentType: params.doc?.metadata?.contentType ?? null,
+ numPages: params.doc?.metadata?.numPages ?? null,
+ creditsUsed: params.creditsUsed ?? null,
},
});
@@ -352,6 +410,9 @@ async function diffAndPersistPage(params: {
status_code: getDocumentStatusCode(params.doc),
metadata: {
title: params.doc?.metadata?.title ?? null,
+ contentType: params.doc?.metadata?.contentType ?? null,
+ numPages: params.doc?.metadata?.numPages ?? null,
+ creditsUsed: params.creditsUsed ?? null,
},
judgment,
emailStatus: status,
@@ -388,6 +449,7 @@ async function runScrapeTarget(params: {
scrapeId: result.scrapeId,
doc: result.doc,
source: "explicit",
+ creditsUsed: result.credits,
}),
);
} catch (error) {
@@ -534,7 +596,8 @@ async function runCrawlTarget(params: {
if (!doc) continue;
const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);
seen.add(hashMonitorUrl(url));
- credits += estimateActualCredits(doc, body.scrapeOptions);
+ const pageCredits = estimateActualCredits(doc, body.scrapeOptions);
+ credits += pageCredits;
pages.push(
await diffAndPersistPage({
monitor: params.monitor,
@@ -544,6 +607,7 @@ async function runCrawlTarget(params: {
scrapeId: job.id,
doc,
source: "discovered",
+ creditsUsed: pageCredits,
}),
);
}
@@ -1331,7 +1395,10 @@ export async function reconcileRunningMonitorChecks(
countMonitorCheckPages({ checkId: check.id, status: "error" }),
]);
const totalPages = same + changed + newCount + removed + errorCount;
- const actualCredits = totalPages;
+ const actualCredits = await calculateMonitorCheckActualCredits({
+ checkId: check.id,
+ targets: monitor.targets,
+ });
let finalized = await updateMonitorCheck(check.id, {
status: errorCount > 0 ? "partial" : "completed",
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
new file mode 100644
index 000000000..f29cfa788
--- /dev/null
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -0,0 +1,83 @@
+jest.mock("uuid", () => ({
+ v7: () => "test-uuid-v7",
+}));
+
+import {
+ calculateMonitorCheckActualCreditsFromPages,
+ estimateMonitorCreditsPerRun,
+} from "./store";
+import type { MonitorTarget } from "./types";
+
+describe("monitoring store credit helpers", () => {
+ it("estimates goal-enabled scrape monitors from scrape option costs", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://example.com/a", "https://example.com/b"],
+ scrapeOptions: {
+ formats: [{ type: "changeTracking", modes: ["json"] }],
+ proxy: "stealth",
+ },
+ },
+ ];
+
+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(18);
+ expect(estimateMonitorCreditsPerRun(targets, true)).toBe(20);
+ });
+
+ it("adds predictable lockdown costs and judge credits separately", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://example.com/a"],
+ scrapeOptions: {
+ lockdown: true,
+ },
+ },
+ ];
+
+ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(5);
+ expect(estimateMonitorCreditsPerRun(targets, true)).toBe(6);
+ });
+
+ it("uses target options when page rows do not have recorded scrape credits", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://example.com/a"],
+ scrapeOptions: {
+ formats: [{ type: "changeTracking", modes: ["json"] }],
+ proxy: "stealth",
+ },
+ },
+ ];
+
+ expect(
+ calculateMonitorCheckActualCreditsFromPages(
+ [
+ {
+ target_id: "target-1",
+ metadata: {},
+ judgment: { meaningful: true },
+ status: "changed",
+ },
+ ],
+ targets,
+ ),
+ ).toBe(10);
+ });
+
+ it("prefers recorded page usage and does not bill removed pages", () => {
+ expect(
+ calculateMonitorCheckActualCreditsFromPages([
+ { metadata: { creditsUsed: 5 }, judgment: { meaningful: true } },
+ { metadata: { creditsUsed: 1 }, judgment: null },
+ { metadata: {}, judgment: { meaningful: false } },
+ { status: "removed", metadata: {}, judgment: null },
+ ]),
+ ).toBe(9);
+ });
+});
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 2cec0f79b..e014ee292 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -28,7 +28,96 @@ function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {
})) as MonitorTarget[];
}
-function estimateTargetCredits(target: MonitorTarget): number {
+const BASE_SCRAPE_CREDITS_PER_PAGE = 1;
+const JSON_SCRAPE_CREDITS_PER_PAGE = 5;
+const SCRAPE_OPTION_CREDIT_BONUS = 4;
+const JUDGE_CREDITS_PER_PAGE = 1;
+const REMOVED_PAGE_CREDITS = 0;
+const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;
+const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;
+
+function formatType(format: unknown): string | null {
+ if (typeof format === "string") return format;
+ if (
+ format &&
+ typeof format === "object" &&
+ "type" in format &&
+ typeof format.type === "string"
+ ) {
+ return format.type;
+ }
+ return null;
+}
+
+function hasFormatOfType(formats: unknown, type: string): boolean {
+ return (
+ Array.isArray(formats) &&
+ formats.some(format => formatType(format) === type)
+ );
+}
+
+function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {
+ return types.some(type => hasFormatOfType(formats, type));
+}
+
+function optionBonus(enabled: boolean): number {
+ return enabled ? SCRAPE_OPTION_CREDIT_BONUS : 0;
+}
+
+function requestsJsonChangeTracking(formats: unknown): boolean {
+ if (!Array.isArray(formats)) return false;
+ return formats.some(format => {
+ if (
+ !format ||
+ typeof format !== "object" ||
+ !("type" in format) ||
+ format.type !== "changeTracking"
+ ) {
+ return false;
+ }
+ const modes = "modes" in format ? format.modes : undefined;
+ return Array.isArray(modes) && modes.includes("json");
+ });
+}
+
+function estimateBaseCreditsPerPage(
+ options: MonitorTarget["scrapeOptions"],
+): number {
+ const formats = options?.formats;
+ const usesJsonCredits =
+ hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);
+ const baseCredits = usesJsonCredits
+ ? JSON_SCRAPE_CREDITS_PER_PAGE
+ : BASE_SCRAPE_CREDITS_PER_PAGE + optionBonus(Boolean(options?.lockdown));
+ const formatBonusCount = [
+ hasAnyFormatOfType(formats, ["question", "query"]),
+ hasFormatOfType(formats, "highlights"),
+ hasFormatOfType(formats, "audio"),
+ hasFormatOfType(formats, "video"),
+ ].reduce((count, enabled) => count + (enabled ? 1 : 0), 0);
+ const proxyCredits = optionBonus(
+ options?.proxy === "stealth" || options?.proxy === "enhanced",
+ );
+
+ return (
+ baseCredits + formatBonusCount * SCRAPE_OPTION_CREDIT_BONUS + proxyCredits
+ );
+}
+
+function estimateTargetBaseCredits(target: MonitorTarget): number {
+ const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);
+ if (target.type === "scrape") {
+ return target.urls.length * creditsPerPage;
+ }
+
+ const limit =
+ typeof target.crawlOptions?.limit === "number"
+ ? target.crawlOptions.limit
+ : DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE;
+ return Math.max(1, limit) * creditsPerPage;
+}
+
+function estimateTargetPageCount(target: MonitorTarget): number {
if (target.type === "scrape") {
return target.urls.length;
}
@@ -36,7 +125,7 @@ function estimateTargetCredits(target: MonitorTarget): number {
const limit =
typeof target.crawlOptions?.limit === "number"
? target.crawlOptions.limit
- : 10000;
+ : DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE;
return Math.max(1, limit);
}
@@ -44,11 +133,51 @@ export function estimateMonitorCreditsPerRun(
targets: MonitorTarget[],
judgeEnabled: boolean = false,
): number {
- const scrapeCredits = targets.reduce(
- (sum, target) => sum + estimateTargetCredits(target),
+ const baseCredits = targets.reduce(
+ (sum, target) => sum + estimateTargetBaseCredits(target),
0,
);
- return judgeEnabled ? scrapeCredits * 2 : scrapeCredits;
+ const judgeCredits = judgeEnabled
+ ? targets.reduce(
+ (sum, target) =>
+ sum + estimateTargetPageCount(target) * JUDGE_CREDITS_PER_PAGE,
+ 0,
+ )
+ : 0;
+ return baseCredits + judgeCredits;
+}
+
+export function calculateMonitorCheckActualCreditsFromPages(
+ pages: Array<{
+ target_id?: string | null;
+ metadata?: unknown;
+ judgment?: unknown;
+ status?: string;
+ }>,
+ targets: MonitorTarget[] = [],
+): number {
+ const baseCreditsByTarget = new Map(
+ targets.map(target => [
+ target.id,
+ estimateBaseCreditsPerPage(target.scrapeOptions),
+ ]),
+ );
+
+ return pages.reduce((total, page) => {
+ const metadata = page.metadata as { creditsUsed?: unknown } | null;
+ const recordedCredits = metadata?.creditsUsed;
+ const baseCredits =
+ typeof recordedCredits === "number" && Number.isFinite(recordedCredits)
+ ? recordedCredits
+ : page.status === "removed"
+ ? REMOVED_PAGE_CREDITS
+ : page.status === "error"
+ ? BASE_SCRAPE_CREDITS_PER_PAGE
+ : (baseCreditsByTarget.get(page.target_id ?? "") ??
+ BASE_SCRAPE_CREDITS_PER_PAGE);
+ const judgeCredits = page.judgment != null ? JUDGE_CREDITS_PER_PAGE : 0;
+ return total + baseCredits + judgeCredits;
+ }, 0);
}
function toMonitorSummary(check: MonitorCheckRow): MonitorSummary {
@@ -566,6 +695,32 @@ export async function countMonitorCheckPages(params: {
return total;
}
+export async function calculateMonitorCheckActualCredits(params: {
+ checkId: string;
+ targets: MonitorTarget[];
+}): Promise<number> {
+ let total = 0;
+ let offset = 0;
+
+ while (true) {
+ const { data, error } = await supabase_rr_service
+ .from("monitor_check_pages")
+ .select("target_id, metadata, judgment, status")
+ .eq("check_id", params.checkId)
+ .range(offset, offset + MONITOR_CHECK_PAGE_BATCH_SIZE - 1);
+
+ throwIfError(error, "Failed to calculate monitor check credits");
+
+ const batch = data ?? [];
+ total += calculateMonitorCheckActualCreditsFromPages(batch, params.targets);
+
+ if (batch.length < MONITOR_CHECK_PAGE_BATCH_SIZE) break;
+ offset += MONITOR_CHECK_PAGE_BATCH_SIZE;
+ }
+
+ return total;
+}
+
export async function getMonitorPage(params: {
monitorId: string;
targetId: string;