Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Refactor scrape credit calc & billing flow
Developers Digest7b3f6774dd54
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex 2a1c87fd8..faf8beca6 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -7,40 +7,209 @@ import {isMonitorCheckStale,MONITOR_CHECK_STALE_TIMEOUT_MS,} from "./runner";+import { calculateScrapeJobCredits } from "../worker/scrape-worker";+import { CostTracking } from "../../lib/cost-tracking";+import { TransportableError } from "../../lib/error";++function monitorScrapeJob(+ params: {+ scrapeOptions?: Record<string, any>;+ internalOptions?: Record<string, any>;+ isScrape?: boolean;+ } = {},+) {+ return {+ id: "job-1",+ data: {+ team_id: "team-1",+ scrapeOptions: {+ formats: [{ type: "markdown" }],+ ...(params.scrapeOptions ?? {}),+ },+ internalOptions: {+ teamId: "team-1",+ zeroDataRetention: false,+ bypassBilling: true,+ ...(params.internalOptions ?? {}),+ },+ is_scrape: params.isScrape,+ },+ } as any;+}++async function monitorScrapeCredits(+ params: {+ scrapeOptions?: Record<string, any>;+ internalOptions?: Record<string, any>;+ metadata?: Record<string, any>;+ flags?: Record<string, any>;+ unsupportedFeatures?: Set<any>;+ } = {},+) {+ return await calculateScrapeJobCredits(+ monitorScrapeJob({+ scrapeOptions: params.scrapeOptions,+ internalOptions: params.internalOptions,+ }),+ { metadata: params.metadata ?? { statusCode: 200 } } as any,+ new CostTracking(),+ (params.flags ?? {}) as any,+ null,+ params.unsupportedFeatures,+ );+}describe("monitoring runner", () => {describe("estimateActualCredits", () => {- it("prefers scrape-reported credits when present", async () => {+ it("prefers scrape-reported credits when present", () => {+ expect(estimateActualCredits({ metadata: { creditsUsed: 9 } })).toBe(9);+ });++ it("falls back to one credit when scrape metadata is missing credits", () => {+ expect(estimateActualCredits({ metadata: { numPages: 4 } })).toBe(1);+ });+ });++ describe("calculateScrapeJobCredits", () => {+ it.each([+ ["base scrape", {}, {}, 1],+ ["lockdown", { lockdown: true }, {}, 5],+ ["json format", { formats: [{ type: "json" }] }, {}, 5],+ [+ "change tracking json mode",+ { formats: [{ type: "changeTracking", modes: ["json"] }] },+ {},+ 5,+ ],+ ["question format", { formats: [{ type: "question" }] }, {}, 5],+ ["query format", { formats: [{ type: "query" }] }, {}, 5],+ ["highlights format", { formats: [{ type: "highlights" }] }, {}, 5],+ ["audio format", { formats: [{ type: "audio" }] }, {}, 5],+ ["video format", { formats: [{ type: "video" }] }, {}, 5],+ ["ten-page parsed PDF", {}, { numPages: 10 }, 10],+ ["PDF parsing disabled", { parsers: [] }, { numPages: 10 }, 1],+ [+ "json plus three-page PDF",+ { formats: [{ type: "json" }] },+ { numPages: 3 },+ 7,+ ],+ ["stealth proxy actually used", {}, { proxyUsed: "stealth" }, 5],+ [+ "X/Twitter postprocessor",+ {},+ { postprocessorsUsed: ["x-twitter"], proxyUsed: "basic" },+ 30,+ ],+ ])(+ "matches scrape billing for %s",+ async (_name, scrapeOptions, metadata, expected) => {+ await expect(+ monitorScrapeCredits({ scrapeOptions, metadata }),+ ).resolves.toBe(expected);+ },+ );++ it("stacks independent format bonuses", async () => {await expect(- estimateActualCredits({- doc: { metadata: { creditsUsed: 9 } },- options: {},- internalOptions: { teamFlags: null },- teamId: "team-1",+ monitorScrapeCredits({+ scrapeOptions: {+ formats: [+ { type: "markdown" },+ { type: "query" },+ { type: "highlights" },+ { type: "audio" },+ { type: "video" },+ ],+ },}),- ).resolves.toBe(9);+ ).resolves.toBe(17);});- it("bills parsed PDFs by extracted page count", async () => {+ it("does not charge stealth proxy bonus when unsupported", async () => {await expect(- estimateActualCredits({- doc: { metadata: { numPages: 4 } },- options: {},- internalOptions: { teamFlags: null },- teamId: "team-1",+ monitorScrapeCredits({+ metadata: { proxyUsed: "stealth" },+ unsupportedFeatures: new Set(["stealthProxy"]),+ }),+ ).resolves.toBe(1);+ });++ it("uses team-specific ZDR pricing when ZDR is enabled", async () => {+ await expect(+ monitorScrapeCredits({+ internalOptions: { zeroDataRetention: true },+ flags: { zdrCost: 3 },}),).resolves.toBe(4);});- it("adds PDF page credits on top of JSON scrape credits", async () => {+ it("does not add ZDR pricing on top of lockdown", async () => {await expect(- estimateActualCredits({- doc: { metadata: { numPages: 3 } },- options: { formats: [{ type: "json" }] },- internalOptions: { teamFlags: null },- teamId: "team-1",+ monitorScrapeCredits({+ scrapeOptions: { lockdown: true },+ internalOptions: { zeroDataRetention: true },+ flags: { zdrCost: 3 },}),- ).resolves.toBe(7);+ ).resolves.toBe(5);+ });++ it("uses cost tracking for Fire-1 billing", async () => {+ const costTracking = new CostTracking();+ costTracking.addCall({+ type: "other",+ metadata: {},+ cost: 0.01,+ model: "fire-1",+ });++ await expect(+ calculateScrapeJobCredits(+ monitorScrapeJob({+ internalOptions: { v1Agent: { model: "fire-1" } },+ }),+ { metadata: { statusCode: 200 } } as any,+ costTracking,+ {} as any,+ ),+ ).resolves.toBe(18);+ });++ it("still computes credits for bypassed monitor scrapes", async () => {+ await expect(+ calculateScrapeJobCredits(+ monitorScrapeJob({+ scrapeOptions: { formats: [{ type: "json" }] },+ internalOptions: { bypassBilling: true },+ }),+ { metadata: { statusCode: 200 } } as any,+ new CostTracking(),+ {} as any,+ ),+ ).resolves.toBe(5);+ });++ it("does not compute credits for scrape jobs that explicitly opt out", async () => {+ await expect(+ calculateScrapeJobCredits(+ monitorScrapeJob({ isScrape: true }),+ { metadata: { statusCode: 200 } } as any,+ new CostTracking(),+ {} as any,+ ),+ ).resolves.toBeNull();+ });++ it("bills DNS failures consistently with scrape billing", async () => {+ await expect(+ calculateScrapeJobCredits(+ monitorScrapeJob(),+ null,+ new CostTracking(),+ {} as any,+ new TransportableError("SCRAPE_DNS_RESOLUTION_ERROR"),+ ),+ ).resolves.toBe(1);});});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 3f9b0ef7b..0d765bfd5 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -7,9 +7,6 @@ import { processJobInternal } from "../worker/scrape-worker";import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";import { ScrapeJobData } from "../../types";import { getJobFromGCS } from "../../lib/gcs-jobs";-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";-import { CostTracking } from "../../lib/cost-tracking";-import { getACUCTeam } from "../../controllers/auth";import { computeAndPersistPageDiff } from "./diff-orchestrator";import { normalizeMonitorFormats } from "./diff";import { autumnService } from "../autumn/autumn.service";@@ -215,29 +212,11 @@ function getDocumentStatusCode(doc: any): number | null {: null;}-export async function estimateActualCredits(params: {- doc: any;- options: any;- internalOptions: any;- teamId: string;-}): Promise<number> {- const { doc, options, internalOptions, teamId } = params;- if (typeof doc?.metadata?.creditsUsed === "number") {- return doc.metadata.creditsUsed;- }-- const flags =- internalOptions && "teamFlags" in internalOptions- ? internalOptions.teamFlags- : ((await getACUCTeam(teamId))?.flags ?? null);-- return await calculateCreditsToBeBilled(- options,- internalOptions,- doc,- new CostTracking(),- flags,- );+export function estimateActualCredits(doc: any): number {+ const creditsUsed = doc?.metadata?.creditsUsed;+ return typeof creditsUsed === "number" && Number.isFinite(creditsUsed)+ ? creditsUsed+ : 1;}async function runSingleScrape(params: {@@ -271,7 +250,6 @@ async function runSingleScrape(params: {saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,bypassBilling: true,zeroDataRetention: false,- teamFlags: (await getACUCTeam(params.monitor.team_id))?.flags ?? null,};const job: NuQJob<ScrapeJobData> = {id: scrapeId,@@ -298,12 +276,7 @@ async function runSingleScrape(params: {return {scrapeId,doc,- credits: await estimateActualCredits({- doc,- options: scrapeOptions,- internalOptions,- teamId: params.monitor.team_id,- }),+ credits: estimateActualCredits(doc),};}@@ -496,7 +469,6 @@ async function runCrawlTarget(params: {saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,zeroDataRetention: false,bypassBilling: true,- teamFlags: (await getACUCTeam(params.monitor.team_id))?.flags ?? null,},team_id: params.monitor.team_id,createdAt: Date.now(),@@ -568,12 +540,7 @@ async function runCrawlTarget(params: {if (!doc) continue;const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);seen.add(hashMonitorUrl(url));- const pageCredits = await estimateActualCredits({- doc,- options: body.scrapeOptions,- internalOptions: sc.internalOptions,- teamId: params.monitor.team_id,- });+ const pageCredits = estimateActualCredits(doc);credits += pageCredits;pages.push(await diffAndPersistPage({diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex e014ee292..d814e896f 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -60,10 +60,6 @@ function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {return types.some(type => hasFormatOfType(formats, type));}-function optionBonus(enabled: boolean): number {- return enabled ? SCRAPE_OPTION_CREDIT_BONUS : 0;-}-function requestsJsonChangeTracking(formats: unknown): boolean {if (!Array.isArray(formats)) return false;return formats.some(format => {@@ -86,22 +82,37 @@ function estimateBaseCreditsPerPage(const formats = options?.formats;const usesJsonCredits =hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);- const baseCredits = usesJsonCredits- ? JSON_SCRAPE_CREDITS_PER_PAGE- : BASE_SCRAPE_CREDITS_PER_PAGE + optionBonus(Boolean(options?.lockdown));- const formatBonusCount = [- hasAnyFormatOfType(formats, ["question", "query"]),- hasFormatOfType(formats, "highlights"),- hasFormatOfType(formats, "audio"),- hasFormatOfType(formats, "video"),- ].reduce((count, enabled) => count + (enabled ? 1 : 0), 0);- const proxyCredits = optionBonus(- options?.proxy === "stealth" || options?.proxy === "enhanced",- );+ let credits = BASE_SCRAPE_CREDITS_PER_PAGE;- return (- baseCredits + formatBonusCount * SCRAPE_OPTION_CREDIT_BONUS + proxyCredits- );+ if (options?.lockdown) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (usesJsonCredits) {+ credits = JSON_SCRAPE_CREDITS_PER_PAGE;+ }++ if (hasAnyFormatOfType(formats, ["question", "query"])) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (hasFormatOfType(formats, "highlights")) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (hasFormatOfType(formats, "audio")) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (hasFormatOfType(formats, "video")) {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ if (options?.proxy === "stealth" || options?.proxy === "enhanced") {+ credits += SCRAPE_OPTION_CREDIT_BONUS;+ }++ return credits;}function estimateTargetBaseCredits(target: MonitorTarget): number {diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex cfe39abae..4bcba3930 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -107,7 +107,14 @@ async function billScrapeJob(error?: Error | null,unsupportedFeatures?: Set<FeatureFlag>,) {- let creditsToBeBilled: number | null = null;+ const creditsToBeBilled = await calculateScrapeJobCredits(+ job,+ document,+ costTracking,+ flags,+ error,+ unsupportedFeatures,+ );const billing = resolveBillingMetadata({billing: job.data.billing,crawlId: job.data.crawl_id,@@ -120,83 +127,93 @@ async function billScrapeJob(};let trackedInRequest = false;- if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {- creditsToBeBilled = await calculateCreditsToBeBilled(- job.data.scrapeOptions,- job.data.internalOptions,- document,- costTracking,- flags,- error,- unsupportedFeatures,- );+ if (creditsToBeBilled === null || job.data.internalOptions?.bypassBilling) {+ return creditsToBeBilled;+ }- if (- job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&- config.USE_DB_AUTHENTICATION- ) {- try {- trackedInRequest = await autumnService.trackCredits({+ if (+ job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&+ config.USE_DB_AUTHENTICATION+ ) {+ try {+ trackedInRequest = await autumnService.trackCredits({+ teamId: job.data.team_id,+ value: creditsToBeBilled,+ properties: autumnProperties,+ requestScoped: true,+ });+ const billingJobId = uuidv7();+ logger.debug(`Adding billing job to queue for team ${job.data.team_id}`, {+ billingJobId,+ billing,+ credits: creditsToBeBilled,+ is_extract: false,+ });++ // Add directly to the billing queue - the billing worker will handle the rest+ await getBillingQueue().add(+ "bill_team",+ {+ team_id: job.data.team_id,+ subscription_id: undefined,+ credits: creditsToBeBilled,+ billing,+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: job.id,+ api_key_id: job.data.apiKeyId,+ autumnTrackInRequest: trackedInRequest,+ },+ {+ jobId: billingJobId,+ priority: 10,+ },+ );++ return creditsToBeBilled;+ } catch (error) {+ logger.error(+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,+ { error },+ );+ if (trackedInRequest && creditsToBeBilled !== null) {+ await autumnService.refundCredits({teamId: job.data.team_id,value: creditsToBeBilled,properties: autumnProperties,- requestScoped: true,});- const billingJobId = uuidv7();- logger.debug(- `Adding billing job to queue for team ${job.data.team_id}`,- {- billingJobId,- billing,- credits: creditsToBeBilled,- is_extract: false,- },- );-- // Add directly to the billing queue - the billing worker will handle the rest- await getBillingQueue().add(- "bill_team",- {- team_id: job.data.team_id,- subscription_id: undefined,- credits: creditsToBeBilled,- billing,- is_extract: false,- timestamp: new Date().toISOString(),- originating_job_id: job.id,- api_key_id: job.data.apiKeyId,- autumnTrackInRequest: trackedInRequest,- },- {- jobId: billingJobId,- priority: 10,- },- );-- return creditsToBeBilled;- } catch (error) {- logger.error(- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,- { error },- );- if (trackedInRequest && creditsToBeBilled !== null) {- await autumnService.refundCredits({- teamId: job.data.team_id,- value: creditsToBeBilled,- properties: autumnProperties,- });- }- captureExceptionWithZdrCheck(error, {- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },- });- return creditsToBeBilled;}+ captureExceptionWithZdrCheck(error, {+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },+ });+ return creditsToBeBilled;}}return creditsToBeBilled;}+export async function calculateScrapeJobCredits(+ job: NuQJob<any>,+ document: Document | null,+ costTracking: CostTracking,+ flags: TeamFlags,+ error?: Error | null,+ unsupportedFeatures?: Set<FeatureFlag>,+): Promise<number | null> {+ if (job.data.is_scrape === true) return null;++ return await calculateCreditsToBeBilled(+ job.data.scrapeOptions,+ job.data.internalOptions ?? { teamId: job.data.team_id },+ document,+ costTracking,+ flags,+ error,+ unsupportedFeatures,+ );+}+async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const logger = _logger.child({module: "queue-worker",