Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Refactor monitoring credit accounting and metadata
Developers Digest128a619aff3e
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex aa8fcc120..6398948ea 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -2,7 +2,6 @@ import { NuQJob } from "../worker/nuq";import { ScrapeJobData } from "../../types";import { logger as _logger } from "../../lib/logger";import { createWebhookSender, WebhookEvent } from "../webhook";-import { billTeam } from "../billing/credit_billing";import { computeAndPersistPageDiff } from "./diff-orchestrator";import { derivePageIsMeaningful } from "./page-events";import {@@ -13,8 +12,6 @@ import {upsertMonitorPage,} from "./store";-const JUDGE_CREDIT_COST = 1;-const logger = _logger.child({ module: "monitoring-results" });function getDocumentUrl(doc: any, fallback: string): string {@@ -174,6 +171,8 @@ export async function recordMonitorScrapeSuccess(metadata: {title: doc?.metadata?.title ?? null,statusCode: getDocumentStatusCode(doc),+ contentType: doc?.metadata?.contentType ?? null,+ numPages: doc?.metadata?.numPages ?? null,creditsUsed: doc?.metadata?.creditsUsed ?? null,},});@@ -195,6 +194,8 @@ export async function recordMonitorScrapeSuccess(status_code: getDocumentStatusCode(doc),metadata: {title: doc?.metadata?.title ?? null,+ contentType: doc?.metadata?.contentType ?? null,+ numPages: doc?.metadata?.numPages ?? null,creditsUsed: doc?.metadata?.creditsUsed ?? null,},judgment: judgment ?? null,@@ -213,25 +214,6 @@ export async function recordMonitorScrapeSuccess(judgmentMeaningful: judgment?.meaningful,});- if (judgment) {- try {- await billTeam(- job.data.team_id,- undefined,- JUDGE_CREDIT_COST,- job.data.apiKeyId ?? null,- { endpoint: "monitor", jobId: monitoring.checkId },- logger,- );- } catch (error) {- logger.warn("Failed to bill judge credit", {- error,- monitorId: monitoring.monitorId,- checkId: monitoring.checkId,- });- }- }-await sendMonitorPageWebhook({teamId: job.data.team_id,monitorId: monitoring.monitorId,diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex 217925c92..c728ea89a 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -2,9 +2,34 @@ jest.mock("uuid", () => ({v7: () => "test-uuid-v7",}));-import { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS } from "./runner";+import {+ estimateActualCredits,+ isMonitorCheckStale,+ MONITOR_CHECK_STALE_TIMEOUT_MS,+} from "./runner";describe("monitoring runner", () => {+ describe("estimateActualCredits", () => {+ it("prefers scrape-reported credits when present", () => {+ expect(estimateActualCredits({ metadata: { creditsUsed: 9 } }, {})).toBe(+ 9,+ );+ });++ it("bills parsed PDFs by extracted page count", () => {+ expect(estimateActualCredits({ metadata: { numPages: 4 } }, {})).toBe(4);+ });++ it("adds PDF page credits on top of JSON scrape credits", () => {+ expect(+ estimateActualCredits(+ { metadata: { numPages: 3 } },+ { formats: ["json"] },+ ),+ ).toBe(7);+ });+ });+describe("isMonitorCheckStale", () => {const now = new Date("2026-05-06T12:00:00.000Z");diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 25c1d94d1..fe2035491 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -23,11 +23,13 @@ import {type ScrapeOptions,crawlRequestSchema,scrapeRequestSchema,+ shouldParsePDF,toV0CrawlerOptions,} from "../../controllers/v2/types";import { createWebhookSender, WebhookEvent } from "../webhook";import { sendMonitoringEmailSummary } from "../notification/monitoring_email";import {+ calculateMonitorCheckActualCredits,getMonitorCheck,getMonitorForUpdate,getMonitorPage,@@ -64,6 +66,10 @@ export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;const MONITOR_CHECK_PAGE_SCAN_LIMIT = 100_000;+const BASE_SCRAPE_CREDITS_PER_DOCUMENT = 1;+const JSON_SCRAPE_CREDITS_PER_DOCUMENT = 5;+const SCRAPE_OPTION_CREDIT_BONUS = 4;+const PDF_CREDITS_PER_PAGE = 1;const TERMINAL_CHECK_STATUSES = new Set(["completed","partial",@@ -211,15 +217,63 @@ function getDocumentStatusCode(doc: any): number | null {: null;}-function estimateActualCredits(doc: any, options: any): number {+function hasScrapeFormat(formats: unknown[], type: string): boolean {+ return formats.some(format =>+ typeof format === "string"+ ? format === type+ : (format as any)?.type === type,+ );+}++function hasJsonChangeTracking(format: unknown): boolean {+ return (+ typeof format === "object" &&+ format !== null &&+ (format as any).type === "changeTracking" &&+ Array.isArray((format as any).modes) &&+ (format as any).modes.includes("json")+ );+}++export function estimateActualCredits(doc: any, options: any): number {if (typeof doc?.metadata?.creditsUsed === "number") {return doc.metadata.creditsUsed;}const formats = Array.isArray(options?.formats) ? options.formats : [];- const hasJson = formats.some((format: any) =>- typeof format === "string" ? format === "json" : format?.type === "json",+ const hasJson =+ hasScrapeFormat(formats, "json") || formats.some(hasJsonChangeTracking);+ const baseCredits = hasJson+ ? JSON_SCRAPE_CREDITS_PER_DOCUMENT+ : BASE_SCRAPE_CREDITS_PER_DOCUMENT ++ (options?.lockdown ? SCRAPE_OPTION_CREDIT_BONUS : 0);+ const formatBonusCount = [+ hasScrapeFormat(formats, "question") || hasScrapeFormat(formats, "query"),+ hasScrapeFormat(formats, "highlights"),+ hasScrapeFormat(formats, "audio"),+ hasScrapeFormat(formats, "video"),+ ].reduce((count, enabled) => count + (enabled ? 1 : 0), 0);+ const pdfPages =+ typeof doc?.metadata?.numPages === "number" &&+ Number.isFinite(doc.metadata.numPages)+ ? Math.max(1, Math.ceil(doc.metadata.numPages))+ : 1;+ const pdfCredits =+ shouldParsePDF(options?.parsers) && pdfPages > 1+ ? PDF_CREDITS_PER_PAGE * (pdfPages - 1)+ : 0;+ const proxyCredits =+ doc?.metadata?.proxyUsed === "stealth" ||+ options?.proxy === "stealth" ||+ options?.proxy === "enhanced"+ ? SCRAPE_OPTION_CREDIT_BONUS+ : 0;++ return (+ baseCredits ++ formatBonusCount * SCRAPE_OPTION_CREDIT_BONUS ++ pdfCredits ++ proxyCredits);- return hasJson ? 5 : 1;}async function runSingleScrape(params: {@@ -290,6 +344,7 @@ async function diffAndPersistPage(params: {scrapeId: string;doc: any;source: "explicit" | "discovered";+ creditsUsed?: number;}): Promise<PageResult> {const previous = await getMonitorPage({monitorId: params.monitor.id,@@ -333,6 +388,9 @@ async function diffAndPersistPage(params: {metadata: {title: params.doc?.metadata?.title ?? null,statusCode: getDocumentStatusCode(params.doc),+ contentType: params.doc?.metadata?.contentType ?? null,+ numPages: params.doc?.metadata?.numPages ?? null,+ creditsUsed: params.creditsUsed ?? null,},});@@ -352,6 +410,9 @@ async function diffAndPersistPage(params: {status_code: getDocumentStatusCode(params.doc),metadata: {title: params.doc?.metadata?.title ?? null,+ contentType: params.doc?.metadata?.contentType ?? null,+ numPages: params.doc?.metadata?.numPages ?? null,+ creditsUsed: params.creditsUsed ?? null,},judgment,emailStatus: status,@@ -388,6 +449,7 @@ async function runScrapeTarget(params: {scrapeId: result.scrapeId,doc: result.doc,source: "explicit",+ creditsUsed: result.credits,}),);} catch (error) {@@ -534,7 +596,8 @@ async function runCrawlTarget(params: {if (!doc) continue;const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);seen.add(hashMonitorUrl(url));- credits += estimateActualCredits(doc, body.scrapeOptions);+ const pageCredits = estimateActualCredits(doc, body.scrapeOptions);+ credits += pageCredits;pages.push(await diffAndPersistPage({monitor: params.monitor,@@ -544,6 +607,7 @@ async function runCrawlTarget(params: {scrapeId: job.id,doc,source: "discovered",+ creditsUsed: pageCredits,}),);}@@ -1331,7 +1395,10 @@ export async function reconcileRunningMonitorChecks(countMonitorCheckPages({ checkId: check.id, status: "error" }),]);const totalPages = same + changed + newCount + removed + errorCount;- const actualCredits = totalPages;+ const actualCredits = await calculateMonitorCheckActualCredits({+ checkId: check.id,+ targets: monitor.targets,+ });let finalized = await updateMonitorCheck(check.id, {status: errorCount > 0 ? "partial" : "completed",diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsnew file mode 100644index 000000000..f29cfa788--- /dev/null+++ b/apps/api/src/services/monitoring/store.test.ts@@ -0,0 +1,83 @@+jest.mock("uuid", () => ({+ v7: () => "test-uuid-v7",+}));++import {+ calculateMonitorCheckActualCreditsFromPages,+ estimateMonitorCreditsPerRun,+} from "./store";+import type { MonitorTarget } from "./types";++describe("monitoring store credit helpers", () => {+ it("estimates goal-enabled scrape monitors from scrape option costs", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/a", "https://example.com/b"],+ scrapeOptions: {+ formats: [{ type: "changeTracking", modes: ["json"] }],+ proxy: "stealth",+ },+ },+ ];++ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(18);+ expect(estimateMonitorCreditsPerRun(targets, true)).toBe(20);+ });++ it("adds predictable lockdown costs and judge credits separately", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/a"],+ scrapeOptions: {+ lockdown: true,+ },+ },+ ];++ expect(estimateMonitorCreditsPerRun(targets, false)).toBe(5);+ expect(estimateMonitorCreditsPerRun(targets, true)).toBe(6);+ });++ it("uses target options when page rows do not have recorded scrape credits", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/a"],+ scrapeOptions: {+ formats: [{ type: "changeTracking", modes: ["json"] }],+ proxy: "stealth",+ },+ },+ ];++ expect(+ calculateMonitorCheckActualCreditsFromPages(+ [+ {+ target_id: "target-1",+ metadata: {},+ judgment: { meaningful: true },+ status: "changed",+ },+ ],+ targets,+ ),+ ).toBe(10);+ });++ it("prefers recorded page usage and does not bill removed pages", () => {+ expect(+ calculateMonitorCheckActualCreditsFromPages([+ { metadata: { creditsUsed: 5 }, judgment: { meaningful: true } },+ { metadata: { creditsUsed: 1 }, judgment: null },+ { metadata: {}, judgment: { meaningful: false } },+ { status: "removed", metadata: {}, judgment: null },+ ]),+ ).toBe(9);+ });+});diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 2cec0f79b..e014ee292 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -28,7 +28,96 @@ function ensureTargetIds(targets: Array<Record<string, any>>): MonitorTarget[] {})) as MonitorTarget[];}-function estimateTargetCredits(target: MonitorTarget): number {+const BASE_SCRAPE_CREDITS_PER_PAGE = 1;+const JSON_SCRAPE_CREDITS_PER_PAGE = 5;+const SCRAPE_OPTION_CREDIT_BONUS = 4;+const JUDGE_CREDITS_PER_PAGE = 1;+const REMOVED_PAGE_CREDITS = 0;+const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;+const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;++function formatType(format: unknown): string | null {+ if (typeof format === "string") return format;+ if (+ format &&+ typeof format === "object" &&+ "type" in format &&+ typeof format.type === "string"+ ) {+ return format.type;+ }+ return null;+}++function hasFormatOfType(formats: unknown, type: string): boolean {+ return (+ Array.isArray(formats) &&+ formats.some(format => formatType(format) === type)+ );+}++function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {+ return types.some(type => hasFormatOfType(formats, type));+}++function optionBonus(enabled: boolean): number {+ return enabled ? SCRAPE_OPTION_CREDIT_BONUS : 0;+}++function requestsJsonChangeTracking(formats: unknown): boolean {+ if (!Array.isArray(formats)) return false;+ return formats.some(format => {+ if (+ !format ||+ typeof format !== "object" ||+ !("type" in format) ||+ format.type !== "changeTracking"+ ) {+ return false;+ }+ const modes = "modes" in format ? format.modes : undefined;+ return Array.isArray(modes) && modes.includes("json");+ });+}++function estimateBaseCreditsPerPage(+ options: MonitorTarget["scrapeOptions"],+): number {+ const formats = options?.formats;+ const usesJsonCredits =+ hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);+ const baseCredits = usesJsonCredits+ ? JSON_SCRAPE_CREDITS_PER_PAGE+ : BASE_SCRAPE_CREDITS_PER_PAGE + optionBonus(Boolean(options?.lockdown));+ const formatBonusCount = [+ hasAnyFormatOfType(formats, ["question", "query"]),+ hasFormatOfType(formats, "highlights"),+ hasFormatOfType(formats, "audio"),+ hasFormatOfType(formats, "video"),+ ].reduce((count, enabled) => count + (enabled ? 1 : 0), 0);+ const proxyCredits = optionBonus(+ options?.proxy === "stealth" || options?.proxy === "enhanced",+ );++ return (+ baseCredits + formatBonusCount * SCRAPE_OPTION_CREDIT_BONUS + proxyCredits+ );+}++function estimateTargetBaseCredits(target: MonitorTarget): number {+ const creditsPerPage = estimateBaseCreditsPerPage(target.scrapeOptions);+ if (target.type === "scrape") {+ return target.urls.length * creditsPerPage;+ }++ const limit =+ typeof target.crawlOptions?.limit === "number"+ ? target.crawlOptions.limit+ : DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE;+ return Math.max(1, limit) * creditsPerPage;+}++function estimateTargetPageCount(target: MonitorTarget): number {if (target.type === "scrape") {return target.urls.length;}@@ -36,7 +125,7 @@ function estimateTargetCredits(target: MonitorTarget): number {const limit =typeof target.crawlOptions?.limit === "number"? target.crawlOptions.limit- : 10000;+ : DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE;return Math.max(1, limit);}@@ -44,11 +133,51 @@ export function estimateMonitorCreditsPerRun(targets: MonitorTarget[],judgeEnabled: boolean = false,): number {- const scrapeCredits = targets.reduce(- (sum, target) => sum + estimateTargetCredits(target),+ const baseCredits = targets.reduce(+ (sum, target) => sum + estimateTargetBaseCredits(target),0,);- return judgeEnabled ? scrapeCredits * 2 : scrapeCredits;+ const judgeCredits = judgeEnabled+ ? targets.reduce(+ (sum, target) =>+ sum + estimateTargetPageCount(target) * JUDGE_CREDITS_PER_PAGE,+ 0,+ )+ : 0;+ return baseCredits + judgeCredits;+}++export function calculateMonitorCheckActualCreditsFromPages(+ pages: Array<{+ target_id?: string | null;+ metadata?: unknown;+ judgment?: unknown;+ status?: string;+ }>,+ targets: MonitorTarget[] = [],+): number {+ const baseCreditsByTarget = new Map(+ targets.map(target => [+ target.id,+ estimateBaseCreditsPerPage(target.scrapeOptions),+ ]),+ );++ return pages.reduce((total, page) => {+ const metadata = page.metadata as { creditsUsed?: unknown } | null;+ const recordedCredits = metadata?.creditsUsed;+ const baseCredits =+ typeof recordedCredits === "number" && Number.isFinite(recordedCredits)+ ? recordedCredits+ : page.status === "removed"+ ? REMOVED_PAGE_CREDITS+ : page.status === "error"+ ? BASE_SCRAPE_CREDITS_PER_PAGE+ : (baseCreditsByTarget.get(page.target_id ?? "") ??+ BASE_SCRAPE_CREDITS_PER_PAGE);+ const judgeCredits = page.judgment != null ? JUDGE_CREDITS_PER_PAGE : 0;+ return total + baseCredits + judgeCredits;+ }, 0);}function toMonitorSummary(check: MonitorCheckRow): MonitorSummary {@@ -566,6 +695,32 @@ export async function countMonitorCheckPages(params: {return total;}+export async function calculateMonitorCheckActualCredits(params: {+ checkId: string;+ targets: MonitorTarget[];+}): Promise<number> {+ let total = 0;+ let offset = 0;++ while (true) {+ const { data, error } = await supabase_rr_service+ .from("monitor_check_pages")+ .select("target_id, metadata, judgment, status")+ .eq("check_id", params.checkId)+ .range(offset, offset + MONITOR_CHECK_PAGE_BATCH_SIZE - 1);++ throwIfError(error, "Failed to calculate monitor check credits");++ const batch = data ?? [];+ total += calculateMonitorCheckActualCreditsFromPages(batch, params.targets);++ if (batch.length < MONITOR_CHECK_PAGE_BATCH_SIZE) break;+ offset += MONITOR_CHECK_PAGE_BATCH_SIZE;+ }++ return total;+}+export async function getMonitorPage(params: {monitorId: string;targetId: string;