Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Record proxy/postprocessor metadata & billing refactor
Developers Digestb66872dd3a24
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.tsindex d47d8e8ce..2641ef741 100644--- a/apps/api/src/services/monitoring/results.ts+++ b/apps/api/src/services/monitoring/results.ts@@ -173,6 +173,8 @@ export async function recordMonitorScrapeSuccess(statusCode: getDocumentStatusCode(doc),contentType: doc?.metadata?.contentType ?? null,numPages: doc?.metadata?.numPages ?? null,+ proxyUsed: doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,creditsUsed: doc?.metadata?.creditsUsed ?? null,},});@@ -196,6 +198,8 @@ export async function recordMonitorScrapeSuccess(title: doc?.metadata?.title ?? null,contentType: doc?.metadata?.contentType ?? null,numPages: doc?.metadata?.numPages ?? null,+ proxyUsed: doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,creditsUsed: doc?.metadata?.creditsUsed ?? null,},judgment: judgment ?? null,diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex faf8beca6..9971b7864 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -7,57 +7,6 @@ import {isMonitorCheckStale,MONITOR_CHECK_STALE_TIMEOUT_MS,} from "./runner";-import { calculateScrapeJobCredits } from "../worker/scrape-worker";-import { CostTracking } from "../../lib/cost-tracking";-import { TransportableError } from "../../lib/error";--function monitorScrapeJob(- params: {- scrapeOptions?: Record<string, any>;- internalOptions?: Record<string, any>;- isScrape?: boolean;- } = {},-) {- return {- id: "job-1",- data: {- team_id: "team-1",- scrapeOptions: {- formats: [{ type: "markdown" }],- ...(params.scrapeOptions ?? {}),- },- internalOptions: {- teamId: "team-1",- zeroDataRetention: false,- bypassBilling: true,- ...(params.internalOptions ?? {}),- },- is_scrape: params.isScrape,- },- } as any;-}--async function monitorScrapeCredits(- params: {- scrapeOptions?: Record<string, any>;- internalOptions?: Record<string, any>;- metadata?: Record<string, any>;- flags?: Record<string, any>;- unsupportedFeatures?: Set<any>;- } = {},-) {- return await calculateScrapeJobCredits(- monitorScrapeJob({- scrapeOptions: params.scrapeOptions,- internalOptions: params.internalOptions,- }),- { metadata: params.metadata ?? { statusCode: 200 } } as any,- new CostTracking(),- (params.flags ?? {}) as any,- null,- params.unsupportedFeatures,- );-}describe("monitoring runner", () => {describe("estimateActualCredits", () => {@@ -70,149 +19,6 @@ describe("monitoring runner", () => {});});- describe("calculateScrapeJobCredits", () => {- it.each([- ["base scrape", {}, {}, 1],- ["lockdown", { lockdown: true }, {}, 5],- ["json format", { formats: [{ type: "json" }] }, {}, 5],- [- "change tracking json mode",- { formats: [{ type: "changeTracking", modes: ["json"] }] },- {},- 5,- ],- ["question format", { formats: [{ type: "question" }] }, {}, 5],- ["query format", { formats: [{ type: "query" }] }, {}, 5],- ["highlights format", { formats: [{ type: "highlights" }] }, {}, 5],- ["audio format", { formats: [{ type: "audio" }] }, {}, 5],- ["video format", { formats: [{ type: "video" }] }, {}, 5],- ["ten-page parsed PDF", {}, { numPages: 10 }, 10],- ["PDF parsing disabled", { parsers: [] }, { numPages: 10 }, 1],- [- "json plus three-page PDF",- { formats: [{ type: "json" }] },- { numPages: 3 },- 7,- ],- ["stealth proxy actually used", {}, { proxyUsed: "stealth" }, 5],- [- "X/Twitter postprocessor",- {},- { postprocessorsUsed: ["x-twitter"], proxyUsed: "basic" },- 30,- ],- ])(- "matches scrape billing for %s",- async (_name, scrapeOptions, metadata, expected) => {- await expect(- monitorScrapeCredits({ scrapeOptions, metadata }),- ).resolves.toBe(expected);- },- );-- it("stacks independent format bonuses", async () => {- await expect(- monitorScrapeCredits({- scrapeOptions: {- formats: [- { type: "markdown" },- { type: "query" },- { type: "highlights" },- { type: "audio" },- { type: "video" },- ],- },- }),- ).resolves.toBe(17);- });-- it("does not charge stealth proxy bonus when unsupported", async () => {- await expect(- monitorScrapeCredits({- metadata: { proxyUsed: "stealth" },- unsupportedFeatures: new Set(["stealthProxy"]),- }),- ).resolves.toBe(1);- });-- it("uses team-specific ZDR pricing when ZDR is enabled", async () => {- await expect(- monitorScrapeCredits({- internalOptions: { zeroDataRetention: true },- flags: { zdrCost: 3 },- }),- ).resolves.toBe(4);- });-- it("does not add ZDR pricing on top of lockdown", async () => {- await expect(- monitorScrapeCredits({- scrapeOptions: { lockdown: true },- internalOptions: { zeroDataRetention: true },- flags: { zdrCost: 3 },- }),- ).resolves.toBe(5);- });-- it("uses cost tracking for Fire-1 billing", async () => {- const costTracking = new CostTracking();- costTracking.addCall({- type: "other",- metadata: {},- cost: 0.01,- model: "fire-1",- });-- await expect(- calculateScrapeJobCredits(- monitorScrapeJob({- internalOptions: { v1Agent: { model: "fire-1" } },- }),- { metadata: { statusCode: 200 } } as any,- costTracking,- {} as any,- ),- ).resolves.toBe(18);- });-- it("still computes credits for bypassed monitor scrapes", async () => {- await expect(- calculateScrapeJobCredits(- monitorScrapeJob({- scrapeOptions: { formats: [{ type: "json" }] },- internalOptions: { bypassBilling: true },- }),- { metadata: { statusCode: 200 } } as any,- new CostTracking(),- {} as any,- ),- ).resolves.toBe(5);- });-- it("does not compute credits for scrape jobs that explicitly opt out", async () => {- await expect(- calculateScrapeJobCredits(- monitorScrapeJob({ isScrape: true }),- { metadata: { statusCode: 200 } } as any,- new CostTracking(),- {} as any,- ),- ).resolves.toBeNull();- });-- it("bills DNS failures consistently with scrape billing", async () => {- await expect(- calculateScrapeJobCredits(- monitorScrapeJob(),- null,- new CostTracking(),- {} as any,- new TransportableError("SCRAPE_DNS_RESOLUTION_ERROR"),- ),- ).resolves.toBe(1);- });- });-describe("isMonitorCheckStale", () => {const now = new Date("2026-05-06T12:00:00.000Z");diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 0d765bfd5..ae26cf328 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -334,6 +334,8 @@ async function diffAndPersistPage(params: {statusCode: getDocumentStatusCode(params.doc),contentType: params.doc?.metadata?.contentType ?? null,numPages: params.doc?.metadata?.numPages ?? null,+ proxyUsed: params.doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,creditsUsed: params.creditsUsed ?? null,},});@@ -356,6 +358,8 @@ async function diffAndPersistPage(params: {title: params.doc?.metadata?.title ?? null,contentType: params.doc?.metadata?.contentType ?? null,numPages: params.doc?.metadata?.numPages ?? null,+ proxyUsed: params.doc?.metadata?.proxyUsed ?? null,+ postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,creditsUsed: params.creditsUsed ?? null,},judgment,diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.tsindex 3b64e8e58..b5aee33a4 100644--- a/apps/api/src/services/monitoring/store.test.ts+++ b/apps/api/src/services/monitoring/store.test.ts@@ -70,6 +70,57 @@ describe("monitoring store credit helpers", () => {).toBe(10);});+ it("uses monitor metadata for fallback PDF credits when recorded usage is missing", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://example.com/sample.pdf"],+ scrapeOptions: {},+ },+ ];++ expect(+ calculateMonitorCheckActualCreditsFromPages(+ [+ {+ target_id: "target-1",+ metadata: { numPages: 5 },+ status: "same",+ },+ ],+ targets,+ ),+ ).toBe(5);+ });++ it("uses monitor metadata for fallback proxy and postprocessor credits", () => {+ const targets: MonitorTarget[] = [+ {+ id: "target-1",+ type: "scrape",+ urls: ["https://x.com/firecrawl"],+ scrapeOptions: {},+ },+ ];++ expect(+ calculateMonitorCheckActualCreditsFromPages(+ [+ {+ target_id: "target-1",+ metadata: {+ proxyUsed: "stealth",+ postprocessorsUsed: ["x-twitter"],+ },+ status: "same",+ },+ ],+ targets,+ ),+ ).toBe(34);+ });+it("prefers recorded page usage and does not bill removed pages", () => {expect(calculateMonitorCheckActualCreditsFromPages([diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex ea1b2fd3c..dd6603d6f 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -1,6 +1,7 @@import { createHash } from "crypto";import { v7 as uuidv7 } from "uuid";import { supabase_rr_service, supabase_service } from "../supabase";+import { shouldParsePDF } from "../../controllers/v2/types";import {getNextMonitorRunAt,estimateRunsPerMonth,@@ -33,9 +34,17 @@ const JSON_SCRAPE_CREDITS_PER_PAGE = 5;const SCRAPE_OPTION_CREDIT_BONUS = 4;const JUDGE_CREDITS_PER_PAGE = 1;const REMOVED_PAGE_CREDITS = 0;+const X_TWITTER_POSTPROCESSOR_CREDIT_BONUS = 29;const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;+type MonitorCreditMetadata = {+ creditsUsed?: unknown;+ numPages?: unknown;+ proxyUsed?: unknown;+ postprocessorsUsed?: unknown;+};+function formatType(format: unknown): string | null {if (typeof format === "string") return format;if (@@ -173,6 +182,7 @@ export function calculateMonitorCheckActualCreditsFromPages(estimateBaseCreditsPerPage(target.scrapeOptions),]),);+ const targetsById = new Map(targets.map(target => [target.id, target]));function fallbackBaseCreditsForPage(page: (typeof pages)[number]): number {if (page.status === "removed") {@@ -183,12 +193,40 @@ export function calculateMonitorCheckActualCreditsFromPages(return BASE_SCRAPE_CREDITS_PER_PAGE;}- // Successful new monitor pages should carry scrape-computed credits in- // metadata. This is only for older rows or unusual missing-metadata paths.- return (+ const metadata = page.metadata as MonitorCreditMetadata | null;+ const target = targetsById.get(page.target_id ?? "");+ let credits =baseCreditsByTarget.get(page.target_id ?? "") ??- BASE_SCRAPE_CREDITS_PER_PAGE- );+ BASE_SCRAPE_CREDITS_PER_PAGE;++ // Monitor-specific fallback only: new rows should prefer metadata.creditsUsed+ // when the scrape path provides it. If it is missing, use retained monitor+ // metadata to avoid obvious undercounts for PDFs and special postprocessors.+ if (+ target &&+ shouldParsePDF(target.scrapeOptions?.parsers as any) &&+ typeof metadata?.numPages === "number" &&+ metadata.numPages > 1+ ) {+ credits += metadata.numPages - 1;+ }++ if (+ metadata?.proxyUsed === "stealth" &&+ target?.scrapeOptions?.proxy !== "stealth" &&+ target?.scrapeOptions?.proxy !== "enhanced"+ ) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (+ Array.isArray(metadata?.postprocessorsUsed) &&+ metadata.postprocessorsUsed.includes("x-twitter")+ ) {+ credits += X_TWITTER_POSTPROCESSOR_CREDIT_BONUS;+ }++ return credits;}function judgeCreditsForPage(page: (typeof pages)[number]): number {@@ -202,7 +240,7 @@ export function calculateMonitorCheckActualCreditsFromPages(}return pages.reduce((total, page) => {- const metadata = page.metadata as { creditsUsed?: unknown } | null;+ const metadata = page.metadata as MonitorCreditMetadata | null;const recordedCredits = metadata?.creditsUsed;let baseCredits = fallbackBaseCreditsForPage(page);diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 0c1f895ec..cfe39abae 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -107,14 +107,7 @@ async function billScrapeJob(error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,) {- const creditsToBeBilled = await calculateScrapeJobCredits(- job,- document,- costTracking,- flags,- error,- unsupportedFeatures,- );+ let creditsToBeBilled: number | null = null;const billing = resolveBillingMetadata({billing: job.data.billing,crawlId: job.data.crawl_id,@@ -127,93 +120,83 @@ async function billScrapeJob(};let trackedInRequest = false;- if (creditsToBeBilled === null || job.data.internalOptions?.bypassBilling) {- return creditsToBeBilled;- }-- if (- job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&- config.USE_DB_AUTHENTICATION- ) {- try {- trackedInRequest = await autumnService.trackCredits({- teamId: job.data.team_id,- value: creditsToBeBilled,- properties: autumnProperties,- requestScoped: true,- });- const billingJobId = uuidv7();- logger.debug(`Adding billing job to queue for team ${job.data.team_id}`, {- billingJobId,- billing,- credits: creditsToBeBilled,- is_extract: false,- });-- // Add directly to the billing queue - the billing worker will handle the rest- await getBillingQueue().add(- "bill_team",- {- team_id: job.data.team_id,- subscription_id: undefined,- credits: creditsToBeBilled,- billing,- is_extract: false,- timestamp: new Date().toISOString(),- originating_job_id: job.id,- api_key_id: job.data.apiKeyId,- autumnTrackInRequest: trackedInRequest,- },- {- jobId: billingJobId,- priority: 10,- },- );+ if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {+ creditsToBeBilled = await calculateCreditsToBeBilled(+ job.data.scrapeOptions,+ job.data.internalOptions,+ document,+ costTracking,+ flags,+ error,+ unsupportedFeatures,+ );- return creditsToBeBilled;- } catch (error) {- logger.error(- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,- { error },- );- if (trackedInRequest && creditsToBeBilled !== null) {- await autumnService.refundCredits({+ if (+ job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&+ config.USE_DB_AUTHENTICATION+ ) {+ try {+ trackedInRequest = await autumnService.trackCredits({teamId: job.data.team_id,value: creditsToBeBilled,properties: autumnProperties,+ requestScoped: true,});+ const billingJobId = uuidv7();+ logger.debug(+ `Adding billing job to queue for team ${job.data.team_id}`,+ {+ billingJobId,+ billing,+ credits: creditsToBeBilled,+ is_extract: false,+ },+ );++ // Add directly to the billing queue - the billing worker will handle the rest+ await getBillingQueue().add(+ "bill_team",+ {+ team_id: job.data.team_id,+ subscription_id: undefined,+ credits: creditsToBeBilled,+ billing,+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: job.id,+ api_key_id: job.data.apiKeyId,+ autumnTrackInRequest: trackedInRequest,+ },+ {+ jobId: billingJobId,+ priority: 10,+ },+ );++ return creditsToBeBilled;+ } catch (error) {+ logger.error(+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,+ { error },+ );+ if (trackedInRequest && creditsToBeBilled !== null) {+ await autumnService.refundCredits({+ teamId: job.data.team_id,+ value: creditsToBeBilled,+ properties: autumnProperties,+ });+ }+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },+ });+ return creditsToBeBilled;}- captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },- });- return creditsToBeBilled;}}return creditsToBeBilled;}-export async function calculateScrapeJobCredits(- job: NuQJob<any>,- document: Document | null,- costTracking: CostTracking,- flags: TeamFlags,- error?: Error | null,- unsupportedFeatures?: Set<FeatureFlag>,-): Promise<number | null> {- if (job.data.is_scrape === true) return null;-- return await calculateCreditsToBeBilled(- job.data.scrapeOptions,- job.data.internalOptions ?? { teamId: job.data.team_id },- document,- costTracking,- flags,- error,- unsupportedFeatures,- );-}-async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const logger = _logger.child({module: "queue-worker",@@ -849,7 +832,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {zeroDataRetention: job.data.zeroDataRetention,}).catch(err => logger.warn("Scrape tracking failed", { error: err }));- await recordMonitorScrapeFailure(job, error, credits_billed);+ await recordMonitorScrapeFailure(job, error);return data;} finally {