Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Refactor scrape credit calc & billing flow

Developers Digest7b3f6774dd54
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index 2a1c87fd8..faf8beca6 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -7,40 +7,209 @@ import {
isMonitorCheckStale,
MONITOR_CHECK_STALE_TIMEOUT_MS,
} from "./runner";
+import { calculateScrapeJobCredits } from "../worker/scrape-worker";
+import { CostTracking } from "../../lib/cost-tracking";
+import { TransportableError } from "../../lib/error";
+
+function monitorScrapeJob(
+ params: {
+ scrapeOptions?: Record<string, any>;
+ internalOptions?: Record<string, any>;
+ isScrape?: boolean;
+ } = {},
+) {
+ return {
+ id: "job-1",
+ data: {
+ team_id: "team-1",
+ scrapeOptions: {
+ formats: [{ type: "markdown" }],
+ ...(params.scrapeOptions ?? {}),
+ },
+ internalOptions: {
+ teamId: "team-1",
+ zeroDataRetention: false,
+ bypassBilling: true,
+ ...(params.internalOptions ?? {}),
+ },
+ is_scrape: params.isScrape,
+ },
+ } as any;
+}
+
+async function monitorScrapeCredits(
+ params: {
+ scrapeOptions?: Record<string, any>;
+ internalOptions?: Record<string, any>;
+ metadata?: Record<string, any>;
+ flags?: Record<string, any>;
+ unsupportedFeatures?: Set<any>;
+ } = {},
+) {
+ return await calculateScrapeJobCredits(
+ monitorScrapeJob({
+ scrapeOptions: params.scrapeOptions,
+ internalOptions: params.internalOptions,
+ }),
+ { metadata: params.metadata ?? { statusCode: 200 } } as any,
+ new CostTracking(),
+ (params.flags ?? {}) as any,
+ null,
+ params.unsupportedFeatures,
+ );
+}
describe("monitoring runner", () => {
describe("estimateActualCredits", () => {
- it("prefers scrape-reported credits when present", async () => {
+ it("prefers scrape-reported credits when present", () => {
+ expect(estimateActualCredits({ metadata: { creditsUsed: 9 } })).toBe(9);
+ });
+
+ it("falls back to one credit when scrape metadata is missing credits", () => {
+ expect(estimateActualCredits({ metadata: { numPages: 4 } })).toBe(1);
+ });
+ });
+
+ describe("calculateScrapeJobCredits", () => {
+ it.each([
+ ["base scrape", {}, {}, 1],
+ ["lockdown", { lockdown: true }, {}, 5],
+ ["json format", { formats: [{ type: "json" }] }, {}, 5],
+ [
+ "change tracking json mode",
+ { formats: [{ type: "changeTracking", modes: ["json"] }] },
+ {},
+ 5,
+ ],
+ ["question format", { formats: [{ type: "question" }] }, {}, 5],
+ ["query format", { formats: [{ type: "query" }] }, {}, 5],
+ ["highlights format", { formats: [{ type: "highlights" }] }, {}, 5],
+ ["audio format", { formats: [{ type: "audio" }] }, {}, 5],
+ ["video format", { formats: [{ type: "video" }] }, {}, 5],
+ ["ten-page parsed PDF", {}, { numPages: 10 }, 10],
+ ["PDF parsing disabled", { parsers: [] }, { numPages: 10 }, 1],
+ [
+ "json plus three-page PDF",
+ { formats: [{ type: "json" }] },
+ { numPages: 3 },
+ 7,
+ ],
+ ["stealth proxy actually used", {}, { proxyUsed: "stealth" }, 5],
+ [
+ "X/Twitter postprocessor",
+ {},
+ { postprocessorsUsed: ["x-twitter"], proxyUsed: "basic" },
+ 30,
+ ],
+ ])(
+ "matches scrape billing for %s",
+ async (_name, scrapeOptions, metadata, expected) => {
+ await expect(
+ monitorScrapeCredits({ scrapeOptions, metadata }),
+ ).resolves.toBe(expected);
+ },
+ );
+
+ it("stacks independent format bonuses", async () => {
await expect(
- estimateActualCredits({
- doc: { metadata: { creditsUsed: 9 } },
- options: {},
- internalOptions: { teamFlags: null },
- teamId: "team-1",
+ monitorScrapeCredits({
+ scrapeOptions: {
+ formats: [
+ { type: "markdown" },
+ { type: "query" },
+ { type: "highlights" },
+ { type: "audio" },
+ { type: "video" },
+ ],
+ },
}),
- ).resolves.toBe(9);
+ ).resolves.toBe(17);
});
- it("bills parsed PDFs by extracted page count", async () => {
+ it("does not charge stealth proxy bonus when unsupported", async () => {
await expect(
- estimateActualCredits({
- doc: { metadata: { numPages: 4 } },
- options: {},
- internalOptions: { teamFlags: null },
- teamId: "team-1",
+ monitorScrapeCredits({
+ metadata: { proxyUsed: "stealth" },
+ unsupportedFeatures: new Set(["stealthProxy"]),
+ }),
+ ).resolves.toBe(1);
+ });
+
+ it("uses team-specific ZDR pricing when ZDR is enabled", async () => {
+ await expect(
+ monitorScrapeCredits({
+ internalOptions: { zeroDataRetention: true },
+ flags: { zdrCost: 3 },
}),
).resolves.toBe(4);
});
- it("adds PDF page credits on top of JSON scrape credits", async () => {
+ it("does not add ZDR pricing on top of lockdown", async () => {
await expect(
- estimateActualCredits({
- doc: { metadata: { numPages: 3 } },
- options: { formats: [{ type: "json" }] },
- internalOptions: { teamFlags: null },
- teamId: "team-1",
+ monitorScrapeCredits({
+ scrapeOptions: { lockdown: true },
+ internalOptions: { zeroDataRetention: true },
+ flags: { zdrCost: 3 },
}),
- ).resolves.toBe(7);
+ ).resolves.toBe(5);
+ });
+
+ it("uses cost tracking for Fire-1 billing", async () => {
+ const costTracking = new CostTracking();
+ costTracking.addCall({
+ type: "other",
+ metadata: {},
+ cost: 0.01,
+ model: "fire-1",
+ });
+
+ await expect(
+ calculateScrapeJobCredits(
+ monitorScrapeJob({
+ internalOptions: { v1Agent: { model: "fire-1" } },
+ }),
+ { metadata: { statusCode: 200 } } as any,
+ costTracking,
+ {} as any,
+ ),
+ ).resolves.toBe(18);
+ });
+
+ it("still computes credits for bypassed monitor scrapes", async () => {
+ await expect(
+ calculateScrapeJobCredits(
+ monitorScrapeJob({
+ scrapeOptions: { formats: [{ type: "json" }] },
+ internalOptions: { bypassBilling: true },
+ }),
+ { metadata: { statusCode: 200 } } as any,
+ new CostTracking(),
+ {} as any,
+ ),
+ ).resolves.toBe(5);
+ });
+
+ it("does not compute credits for scrape jobs that explicitly opt out", async () => {
+ await expect(
+ calculateScrapeJobCredits(
+ monitorScrapeJob({ isScrape: true }),
+ { metadata: { statusCode: 200 } } as any,
+ new CostTracking(),
+ {} as any,
+ ),
+ ).resolves.toBeNull();
+ });
+
+ it("bills DNS failures consistently with scrape billing", async () => {
+ await expect(
+ calculateScrapeJobCredits(
+ monitorScrapeJob(),
+ null,
+ new CostTracking(),
+ {} as any,
+ new TransportableError("SCRAPE_DNS_RESOLUTION_ERROR"),
+ ),
+ ).resolves.toBe(1);
});
});
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 3f9b0ef7b..0d765bfd5 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -7,9 +7,6 @@ import { processJobInternal } from "../worker/scrape-worker";
import { NuQJob, crawlGroup, scrapeQueue } from "../worker/nuq";
import { ScrapeJobData } from "../../types";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
-import { CostTracking } from "../../lib/cost-tracking";
-import { getACUCTeam } from "../../controllers/auth";
import { computeAndPersistPageDiff } from "./diff-orchestrator";
import { normalizeMonitorFormats } from "./diff";
import { autumnService } from "../autumn/autumn.service";
@@ -215,29 +212,11 @@ function getDocumentStatusCode(doc: any): number | null {
: null;
}
-export async function estimateActualCredits(params: {
- doc: any;
- options: any;
- internalOptions: any;
- teamId: string;
-}): Promise<number> {
- const { doc, options, internalOptions, teamId } = params;
- if (typeof doc?.metadata?.creditsUsed === "number") {
- return doc.metadata.creditsUsed;
- }
-
- const flags =
- internalOptions && "teamFlags" in internalOptions
- ? internalOptions.teamFlags
- : ((await getACUCTeam(teamId))?.flags ?? null);
-
- return await calculateCreditsToBeBilled(
- options,
- internalOptions,
- doc,
- new CostTracking(),
- flags,
- );
+export function estimateActualCredits(doc: any): number {
+ const creditsUsed = doc?.metadata?.creditsUsed;
+ return typeof creditsUsed === "number" && Number.isFinite(creditsUsed)
+ ? creditsUsed
+ : 1;
}
async function runSingleScrape(params: {
@@ -271,7 +250,6 @@ async function runSingleScrape(params: {
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
bypassBilling: true,
zeroDataRetention: false,
- teamFlags: (await getACUCTeam(params.monitor.team_id))?.flags ?? null,
};
const job: NuQJob<ScrapeJobData> = {
id: scrapeId,
@@ -298,12 +276,7 @@ async function runSingleScrape(params: {
return {
scrapeId,
doc,
- credits: await estimateActualCredits({
- doc,
- options: scrapeOptions,
- internalOptions,
- teamId: params.monitor.team_id,
- }),
+ credits: estimateActualCredits(doc),
};
}
@@ -496,7 +469,6 @@ async function runCrawlTarget(params: {
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
zeroDataRetention: false,
bypassBilling: true,
- teamFlags: (await getACUCTeam(params.monitor.team_id))?.flags ?? null,
},
team_id: params.monitor.team_id,
createdAt: Date.now(),
@@ -568,12 +540,7 @@ async function runCrawlTarget(params: {
if (!doc) continue;
const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);
seen.add(hashMonitorUrl(url));
- const pageCredits = await estimateActualCredits({
- doc,
- options: body.scrapeOptions,
- internalOptions: sc.internalOptions,
- teamId: params.monitor.team_id,
- });
+ const pageCredits = estimateActualCredits(doc);
credits += pageCredits;
pages.push(
await diffAndPersistPage({
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index e014ee292..d814e896f 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -60,10 +60,6 @@ function hasAnyFormatOfType(formats: unknown, types: string[]): boolean {
return types.some(type => hasFormatOfType(formats, type));
}
-function optionBonus(enabled: boolean): number {
- return enabled ? SCRAPE_OPTION_CREDIT_BONUS : 0;
-}
-
function requestsJsonChangeTracking(formats: unknown): boolean {
if (!Array.isArray(formats)) return false;
return formats.some(format => {
@@ -86,22 +82,37 @@ function estimateBaseCreditsPerPage(
const formats = options?.formats;
const usesJsonCredits =
hasFormatOfType(formats, "json") || requestsJsonChangeTracking(formats);
- const baseCredits = usesJsonCredits
- ? JSON_SCRAPE_CREDITS_PER_PAGE
- : BASE_SCRAPE_CREDITS_PER_PAGE + optionBonus(Boolean(options?.lockdown));
- const formatBonusCount = [
- hasAnyFormatOfType(formats, ["question", "query"]),
- hasFormatOfType(formats, "highlights"),
- hasFormatOfType(formats, "audio"),
- hasFormatOfType(formats, "video"),
- ].reduce((count, enabled) => count + (enabled ? 1 : 0), 0);
- const proxyCredits = optionBonus(
- options?.proxy === "stealth" || options?.proxy === "enhanced",
- );
+ let credits = BASE_SCRAPE_CREDITS_PER_PAGE;
- return (
- baseCredits + formatBonusCount * SCRAPE_OPTION_CREDIT_BONUS + proxyCredits
- );
+ if (options?.lockdown) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (usesJsonCredits) {
+ credits = JSON_SCRAPE_CREDITS_PER_PAGE;
+ }
+
+ if (hasAnyFormatOfType(formats, ["question", "query"])) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (hasFormatOfType(formats, "highlights")) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (hasFormatOfType(formats, "audio")) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (hasFormatOfType(formats, "video")) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (options?.proxy === "stealth" || options?.proxy === "enhanced") {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ return credits;
}
function estimateTargetBaseCredits(target: MonitorTarget): number {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index cfe39abae..4bcba3930 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -107,7 +107,14 @@ async function billScrapeJob(
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
) {
- let creditsToBeBilled: number | null = null;
+ const creditsToBeBilled = await calculateScrapeJobCredits(
+ job,
+ document,
+ costTracking,
+ flags,
+ error,
+ unsupportedFeatures,
+ );
const billing = resolveBillingMetadata({
billing: job.data.billing,
crawlId: job.data.crawl_id,
@@ -120,83 +127,93 @@ async function billScrapeJob(
};
let trackedInRequest = false;
- if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
- creditsToBeBilled = await calculateCreditsToBeBilled(
- job.data.scrapeOptions,
- job.data.internalOptions,
- document,
- costTracking,
- flags,
- error,
- unsupportedFeatures,
- );
+ if (creditsToBeBilled === null || job.data.internalOptions?.bypassBilling) {
+ return creditsToBeBilled;
+ }
- if (
- job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
- config.USE_DB_AUTHENTICATION
- ) {
- try {
- trackedInRequest = await autumnService.trackCredits({
+ if (
+ job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
+ config.USE_DB_AUTHENTICATION
+ ) {
+ try {
+ trackedInRequest = await autumnService.trackCredits({
+ teamId: job.data.team_id,
+ value: creditsToBeBilled,
+ properties: autumnProperties,
+ requestScoped: true,
+ });
+ const billingJobId = uuidv7();
+ logger.debug(`Adding billing job to queue for team ${job.data.team_id}`, {
+ billingJobId,
+ billing,
+ credits: creditsToBeBilled,
+ is_extract: false,
+ });
+
+ // Add directly to the billing queue - the billing worker will handle the rest
+ await getBillingQueue().add(
+ "bill_team",
+ {
+ team_id: job.data.team_id,
+ subscription_id: undefined,
+ credits: creditsToBeBilled,
+ billing,
+ is_extract: false,
+ timestamp: new Date().toISOString(),
+ originating_job_id: job.id,
+ api_key_id: job.data.apiKeyId,
+ autumnTrackInRequest: trackedInRequest,
+ },
+ {
+ jobId: billingJobId,
+ priority: 10,
+ },
+ );
+
+ return creditsToBeBilled;
+ } catch (error) {
+ logger.error(
+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
+ { error },
+ );
+ if (trackedInRequest && creditsToBeBilled !== null) {
+ await autumnService.refundCredits({
teamId: job.data.team_id,
value: creditsToBeBilled,
properties: autumnProperties,
- requestScoped: true,
});
- const billingJobId = uuidv7();
- logger.debug(
- `Adding billing job to queue for team ${job.data.team_id}`,
- {
- billingJobId,
- billing,
- credits: creditsToBeBilled,
- is_extract: false,
- },
- );
-
- // Add directly to the billing queue - the billing worker will handle the rest
- await getBillingQueue().add(
- "bill_team",
- {
- team_id: job.data.team_id,
- subscription_id: undefined,
- credits: creditsToBeBilled,
- billing,
- is_extract: false,
- timestamp: new Date().toISOString(),
- originating_job_id: job.id,
- api_key_id: job.data.apiKeyId,
- autumnTrackInRequest: trackedInRequest,
- },
- {
- jobId: billingJobId,
- priority: 10,
- },
- );
-
- return creditsToBeBilled;
- } catch (error) {
- logger.error(
- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
- { error },
- );
- if (trackedInRequest && creditsToBeBilled !== null) {
- await autumnService.refundCredits({
- teamId: job.data.team_id,
- value: creditsToBeBilled,
- properties: autumnProperties,
- });
- }
- captureExceptionWithZdrCheck(error, {
- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
- });
- return creditsToBeBilled;
}
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
+ });
+ return creditsToBeBilled;
}
}
return creditsToBeBilled;
}
+export async function calculateScrapeJobCredits(
+ job: NuQJob<any>,
+ document: Document | null,
+ costTracking: CostTracking,
+ flags: TeamFlags,
+ error?: Error | null,
+ unsupportedFeatures?: Set<FeatureFlag>,
+): Promise<number | null> {
+ if (job.data.is_scrape === true) return null;
+
+ return await calculateCreditsToBeBilled(
+ job.data.scrapeOptions,
+ job.data.internalOptions ?? { teamId: job.data.team_id },
+ document,
+ costTracking,
+ flags,
+ error,
+ unsupportedFeatures,
+ );
+}
+
async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const logger = _logger.child({
module: "queue-worker",