Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Record proxy/postprocessor metadata & billing refactor

Developers Digestb66872dd3a24
diff --git a/apps/api/src/services/monitoring/results.ts b/apps/api/src/services/monitoring/results.ts
index d47d8e8ce..2641ef741 100644
--- a/apps/api/src/services/monitoring/results.ts
+++ b/apps/api/src/services/monitoring/results.ts
@@ -173,6 +173,8 @@ export async function recordMonitorScrapeSuccess(
statusCode: getDocumentStatusCode(doc),
contentType: doc?.metadata?.contentType ?? null,
numPages: doc?.metadata?.numPages ?? null,
+ proxyUsed: doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,
creditsUsed: doc?.metadata?.creditsUsed ?? null,
},
});
@@ -196,6 +198,8 @@ export async function recordMonitorScrapeSuccess(
title: doc?.metadata?.title ?? null,
contentType: doc?.metadata?.contentType ?? null,
numPages: doc?.metadata?.numPages ?? null,
+ proxyUsed: doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: doc?.metadata?.postprocessorsUsed ?? null,
creditsUsed: doc?.metadata?.creditsUsed ?? null,
},
judgment: judgment ?? null,
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.ts
index faf8beca6..9971b7864 100644
--- a/apps/api/src/services/monitoring/runner.test.ts
+++ b/apps/api/src/services/monitoring/runner.test.ts
@@ -7,57 +7,6 @@ import {
isMonitorCheckStale,
MONITOR_CHECK_STALE_TIMEOUT_MS,
} from "./runner";
-import { calculateScrapeJobCredits } from "../worker/scrape-worker";
-import { CostTracking } from "../../lib/cost-tracking";
-import { TransportableError } from "../../lib/error";
-
-function monitorScrapeJob(
- params: {
- scrapeOptions?: Record<string, any>;
- internalOptions?: Record<string, any>;
- isScrape?: boolean;
- } = {},
-) {
- return {
- id: "job-1",
- data: {
- team_id: "team-1",
- scrapeOptions: {
- formats: [{ type: "markdown" }],
- ...(params.scrapeOptions ?? {}),
- },
- internalOptions: {
- teamId: "team-1",
- zeroDataRetention: false,
- bypassBilling: true,
- ...(params.internalOptions ?? {}),
- },
- is_scrape: params.isScrape,
- },
- } as any;
-}
-
-async function monitorScrapeCredits(
- params: {
- scrapeOptions?: Record<string, any>;
- internalOptions?: Record<string, any>;
- metadata?: Record<string, any>;
- flags?: Record<string, any>;
- unsupportedFeatures?: Set<any>;
- } = {},
-) {
- return await calculateScrapeJobCredits(
- monitorScrapeJob({
- scrapeOptions: params.scrapeOptions,
- internalOptions: params.internalOptions,
- }),
- { metadata: params.metadata ?? { statusCode: 200 } } as any,
- new CostTracking(),
- (params.flags ?? {}) as any,
- null,
- params.unsupportedFeatures,
- );
-}
describe("monitoring runner", () => {
describe("estimateActualCredits", () => {
@@ -70,149 +19,6 @@ describe("monitoring runner", () => {
});
});
- describe("calculateScrapeJobCredits", () => {
- it.each([
- ["base scrape", {}, {}, 1],
- ["lockdown", { lockdown: true }, {}, 5],
- ["json format", { formats: [{ type: "json" }] }, {}, 5],
- [
- "change tracking json mode",
- { formats: [{ type: "changeTracking", modes: ["json"] }] },
- {},
- 5,
- ],
- ["question format", { formats: [{ type: "question" }] }, {}, 5],
- ["query format", { formats: [{ type: "query" }] }, {}, 5],
- ["highlights format", { formats: [{ type: "highlights" }] }, {}, 5],
- ["audio format", { formats: [{ type: "audio" }] }, {}, 5],
- ["video format", { formats: [{ type: "video" }] }, {}, 5],
- ["ten-page parsed PDF", {}, { numPages: 10 }, 10],
- ["PDF parsing disabled", { parsers: [] }, { numPages: 10 }, 1],
- [
- "json plus three-page PDF",
- { formats: [{ type: "json" }] },
- { numPages: 3 },
- 7,
- ],
- ["stealth proxy actually used", {}, { proxyUsed: "stealth" }, 5],
- [
- "X/Twitter postprocessor",
- {},
- { postprocessorsUsed: ["x-twitter"], proxyUsed: "basic" },
- 30,
- ],
- ])(
- "matches scrape billing for %s",
- async (_name, scrapeOptions, metadata, expected) => {
- await expect(
- monitorScrapeCredits({ scrapeOptions, metadata }),
- ).resolves.toBe(expected);
- },
- );
-
- it("stacks independent format bonuses", async () => {
- await expect(
- monitorScrapeCredits({
- scrapeOptions: {
- formats: [
- { type: "markdown" },
- { type: "query" },
- { type: "highlights" },
- { type: "audio" },
- { type: "video" },
- ],
- },
- }),
- ).resolves.toBe(17);
- });
-
- it("does not charge stealth proxy bonus when unsupported", async () => {
- await expect(
- monitorScrapeCredits({
- metadata: { proxyUsed: "stealth" },
- unsupportedFeatures: new Set(["stealthProxy"]),
- }),
- ).resolves.toBe(1);
- });
-
- it("uses team-specific ZDR pricing when ZDR is enabled", async () => {
- await expect(
- monitorScrapeCredits({
- internalOptions: { zeroDataRetention: true },
- flags: { zdrCost: 3 },
- }),
- ).resolves.toBe(4);
- });
-
- it("does not add ZDR pricing on top of lockdown", async () => {
- await expect(
- monitorScrapeCredits({
- scrapeOptions: { lockdown: true },
- internalOptions: { zeroDataRetention: true },
- flags: { zdrCost: 3 },
- }),
- ).resolves.toBe(5);
- });
-
- it("uses cost tracking for Fire-1 billing", async () => {
- const costTracking = new CostTracking();
- costTracking.addCall({
- type: "other",
- metadata: {},
- cost: 0.01,
- model: "fire-1",
- });
-
- await expect(
- calculateScrapeJobCredits(
- monitorScrapeJob({
- internalOptions: { v1Agent: { model: "fire-1" } },
- }),
- { metadata: { statusCode: 200 } } as any,
- costTracking,
- {} as any,
- ),
- ).resolves.toBe(18);
- });
-
- it("still computes credits for bypassed monitor scrapes", async () => {
- await expect(
- calculateScrapeJobCredits(
- monitorScrapeJob({
- scrapeOptions: { formats: [{ type: "json" }] },
- internalOptions: { bypassBilling: true },
- }),
- { metadata: { statusCode: 200 } } as any,
- new CostTracking(),
- {} as any,
- ),
- ).resolves.toBe(5);
- });
-
- it("does not compute credits for scrape jobs that explicitly opt out", async () => {
- await expect(
- calculateScrapeJobCredits(
- monitorScrapeJob({ isScrape: true }),
- { metadata: { statusCode: 200 } } as any,
- new CostTracking(),
- {} as any,
- ),
- ).resolves.toBeNull();
- });
-
- it("bills DNS failures consistently with scrape billing", async () => {
- await expect(
- calculateScrapeJobCredits(
- monitorScrapeJob(),
- null,
- new CostTracking(),
- {} as any,
- new TransportableError("SCRAPE_DNS_RESOLUTION_ERROR"),
- ),
- ).resolves.toBe(1);
- });
- });
-
describe("isMonitorCheckStale", () => {
const now = new Date("2026-05-06T12:00:00.000Z");
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 0d765bfd5..ae26cf328 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -334,6 +334,8 @@ async function diffAndPersistPage(params: {
statusCode: getDocumentStatusCode(params.doc),
contentType: params.doc?.metadata?.contentType ?? null,
numPages: params.doc?.metadata?.numPages ?? null,
+ proxyUsed: params.doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,
creditsUsed: params.creditsUsed ?? null,
},
});
@@ -356,6 +358,8 @@ async function diffAndPersistPage(params: {
title: params.doc?.metadata?.title ?? null,
contentType: params.doc?.metadata?.contentType ?? null,
numPages: params.doc?.metadata?.numPages ?? null,
+ proxyUsed: params.doc?.metadata?.proxyUsed ?? null,
+ postprocessorsUsed: params.doc?.metadata?.postprocessorsUsed ?? null,
creditsUsed: params.creditsUsed ?? null,
},
judgment,
diff --git a/apps/api/src/services/monitoring/store.test.ts b/apps/api/src/services/monitoring/store.test.ts
index 3b64e8e58..b5aee33a4 100644
--- a/apps/api/src/services/monitoring/store.test.ts
+++ b/apps/api/src/services/monitoring/store.test.ts
@@ -70,6 +70,57 @@ describe("monitoring store credit helpers", () => {
).toBe(10);
});
+ it("uses monitor metadata for fallback PDF credits when recorded usage is missing", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://example.com/sample.pdf"],
+ scrapeOptions: {},
+ },
+ ];
+
+ expect(
+ calculateMonitorCheckActualCreditsFromPages(
+ [
+ {
+ target_id: "target-1",
+ metadata: { numPages: 5 },
+ status: "same",
+ },
+ ],
+ targets,
+ ),
+ ).toBe(5);
+ });
+
+ it("uses monitor metadata for fallback proxy and postprocessor credits", () => {
+ const targets: MonitorTarget[] = [
+ {
+ id: "target-1",
+ type: "scrape",
+ urls: ["https://x.com/firecrawl"],
+ scrapeOptions: {},
+ },
+ ];
+
+ expect(
+ calculateMonitorCheckActualCreditsFromPages(
+ [
+ {
+ target_id: "target-1",
+ metadata: {
+ proxyUsed: "stealth",
+ postprocessorsUsed: ["x-twitter"],
+ },
+ status: "same",
+ },
+ ],
+ targets,
+ ),
+ ).toBe(34);
+ });
+
it("prefers recorded page usage and does not bill removed pages", () => {
expect(
calculateMonitorCheckActualCreditsFromPages([
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index ea1b2fd3c..dd6603d6f 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -1,6 +1,7 @@
import { createHash } from "crypto";
import { v7 as uuidv7 } from "uuid";
import { supabase_rr_service, supabase_service } from "../supabase";
+import { shouldParsePDF } from "../../controllers/v2/types";
import {
getNextMonitorRunAt,
estimateRunsPerMonth,
@@ -33,9 +34,17 @@ const JSON_SCRAPE_CREDITS_PER_PAGE = 5;
const SCRAPE_OPTION_CREDIT_BONUS = 4;
const JUDGE_CREDITS_PER_PAGE = 1;
const REMOVED_PAGE_CREDITS = 0;
+const X_TWITTER_POSTPROCESSOR_CREDIT_BONUS = 29;
const DEFAULT_CRAWL_LIMIT_FOR_ESTIMATE = 10000;
const MONITOR_CHECK_PAGE_BATCH_SIZE = 1000;
+type MonitorCreditMetadata = {
+ creditsUsed?: unknown;
+ numPages?: unknown;
+ proxyUsed?: unknown;
+ postprocessorsUsed?: unknown;
+};
+
function formatType(format: unknown): string | null {
if (typeof format === "string") return format;
if (
@@ -173,6 +182,7 @@ export function calculateMonitorCheckActualCreditsFromPages(
estimateBaseCreditsPerPage(target.scrapeOptions),
]),
);
+ const targetsById = new Map(targets.map(target => [target.id, target]));
function fallbackBaseCreditsForPage(page: (typeof pages)[number]): number {
if (page.status === "removed") {
@@ -183,12 +193,40 @@ export function calculateMonitorCheckActualCreditsFromPages(
return BASE_SCRAPE_CREDITS_PER_PAGE;
}
- // Successful new monitor pages should carry scrape-computed credits in
- // metadata. This is only for older rows or unusual missing-metadata paths.
- return (
+ const metadata = page.metadata as MonitorCreditMetadata | null;
+ const target = targetsById.get(page.target_id ?? "");
+ let credits =
baseCreditsByTarget.get(page.target_id ?? "") ??
- BASE_SCRAPE_CREDITS_PER_PAGE
- );
+ BASE_SCRAPE_CREDITS_PER_PAGE;
+
+ // Monitor-specific fallback only: new rows should prefer metadata.creditsUsed
+ // when the scrape path provides it. If it is missing, use retained monitor
+ // metadata to avoid obvious undercounts for PDFs and special postprocessors.
+ if (
+ target &&
+ shouldParsePDF(target.scrapeOptions?.parsers as any) &&
+ typeof metadata?.numPages === "number" &&
+ metadata.numPages > 1
+ ) {
+ credits += metadata.numPages - 1;
+ }
+
+ if (
+ metadata?.proxyUsed === "stealth" &&
+ target?.scrapeOptions?.proxy !== "stealth" &&
+ target?.scrapeOptions?.proxy !== "enhanced"
+ ) {
+ credits += SCRAPE_OPTION_CREDIT_BONUS;
+ }
+
+ if (
+ Array.isArray(metadata?.postprocessorsUsed) &&
+ metadata.postprocessorsUsed.includes("x-twitter")
+ ) {
+ credits += X_TWITTER_POSTPROCESSOR_CREDIT_BONUS;
+ }
+
+ return credits;
}
function judgeCreditsForPage(page: (typeof pages)[number]): number {
@@ -202,7 +240,7 @@ export function calculateMonitorCheckActualCreditsFromPages(
}
return pages.reduce((total, page) => {
- const metadata = page.metadata as { creditsUsed?: unknown } | null;
+ const metadata = page.metadata as MonitorCreditMetadata | null;
const recordedCredits = metadata?.creditsUsed;
let baseCredits = fallbackBaseCreditsForPage(page);
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 0c1f895ec..cfe39abae 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -107,14 +107,7 @@ async function billScrapeJob(
error?: Error | null,
unsupportedFeatures?: Set<FeatureFlag>,
) {
- const creditsToBeBilled = await calculateScrapeJobCredits(
- job,
- document,
- costTracking,
- flags,
- error,
- unsupportedFeatures,
- );
+ let creditsToBeBilled: number | null = null;
const billing = resolveBillingMetadata({
billing: job.data.billing,
crawlId: job.data.crawl_id,
@@ -127,93 +120,83 @@ async function billScrapeJob(
};
let trackedInRequest = false;
- if (creditsToBeBilled === null || job.data.internalOptions?.bypassBilling) {
- return creditsToBeBilled;
- }
-
- if (
- job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
- config.USE_DB_AUTHENTICATION
- ) {
- try {
- trackedInRequest = await autumnService.trackCredits({
- teamId: job.data.team_id,
- value: creditsToBeBilled,
- properties: autumnProperties,
- requestScoped: true,
- });
- const billingJobId = uuidv7();
- logger.debug(`Adding billing job to queue for team ${job.data.team_id}`, {
- billingJobId,
- billing,
- credits: creditsToBeBilled,
- is_extract: false,
- });
-
- // Add directly to the billing queue - the billing worker will handle the rest
- await getBillingQueue().add(
- "bill_team",
- {
- team_id: job.data.team_id,
- subscription_id: undefined,
- credits: creditsToBeBilled,
- billing,
- is_extract: false,
- timestamp: new Date().toISOString(),
- originating_job_id: job.id,
- api_key_id: job.data.apiKeyId,
- autumnTrackInRequest: trackedInRequest,
- },
- {
- jobId: billingJobId,
- priority: 10,
- },
- );
+ if (job.data.is_scrape !== true && !job.data.internalOptions?.bypassBilling) {
+ creditsToBeBilled = await calculateCreditsToBeBilled(
+ job.data.scrapeOptions,
+ job.data.internalOptions,
+ document,
+ costTracking,
+ flags,
+ error,
+ unsupportedFeatures,
+ );
- return creditsToBeBilled;
- } catch (error) {
- logger.error(
- `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
- { error },
- );
- if (trackedInRequest && creditsToBeBilled !== null) {
- await autumnService.refundCredits({
+ if (
+ job.data.team_id !== config.BACKGROUND_INDEX_TEAM_ID! &&
+ config.USE_DB_AUTHENTICATION
+ ) {
+ try {
+ trackedInRequest = await autumnService.trackCredits({
teamId: job.data.team_id,
value: creditsToBeBilled,
properties: autumnProperties,
+ requestScoped: true,
});
+ const billingJobId = uuidv7();
+ logger.debug(
+ `Adding billing job to queue for team ${job.data.team_id}`,
+ {
+ billingJobId,
+ billing,
+ credits: creditsToBeBilled,
+ is_extract: false,
+ },
+ );
+
+ // Add directly to the billing queue - the billing worker will handle the rest
+ await getBillingQueue().add(
+ "bill_team",
+ {
+ team_id: job.data.team_id,
+ subscription_id: undefined,
+ credits: creditsToBeBilled,
+ billing,
+ is_extract: false,
+ timestamp: new Date().toISOString(),
+ originating_job_id: job.id,
+ api_key_id: job.data.apiKeyId,
+ autumnTrackInRequest: trackedInRequest,
+ },
+ {
+ jobId: billingJobId,
+ priority: 10,
+ },
+ );
+
+ return creditsToBeBilled;
+ } catch (error) {
+ logger.error(
+ `Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,
+ { error },
+ );
+ if (trackedInRequest && creditsToBeBilled !== null) {
+ await autumnService.refundCredits({
+ teamId: job.data.team_id,
+ value: creditsToBeBilled,
+ properties: autumnProperties,
+ });
+ }
+ captureExceptionWithZdrCheck(error, {
+ extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
+ });
+ return creditsToBeBilled;
}
- captureExceptionWithZdrCheck(error, {
- extra: { zeroDataRetention: job.data.zeroDataRetention ?? false },
- });
- return creditsToBeBilled;
}
}
return creditsToBeBilled;
}
-export async function calculateScrapeJobCredits(
- job: NuQJob<any>,
- document: Document | null,
- costTracking: CostTracking,
- flags: TeamFlags,
- error?: Error | null,
- unsupportedFeatures?: Set<FeatureFlag>,
-): Promise<number | null> {
- if (job.data.is_scrape === true) return null;
-
- return await calculateCreditsToBeBilled(
- job.data.scrapeOptions,
- job.data.internalOptions ?? { teamId: job.data.team_id },
- document,
- costTracking,
- flags,
- error,
- unsupportedFeatures,
- );
-}
-
async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const logger = _logger.child({
module: "queue-worker",
@@ -849,7 +832,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
zeroDataRetention: job.data.zeroDataRetention,
}).catch(err => logger.warn("Scrape tracking failed", { error: err }));
- await recordMonitorScrapeFailure(job, error, credits_billed);
+ await recordMonitorScrapeFailure(job, error);
return data;
} finally {