Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(billing): drop direct-route failure handling in billing callers (#5482)
Rakshith Ramprakash32119bd8a655
diff --git a/apps/api/src/services/alexandria/README.md b/apps/api/src/services/alexandria/README.mdindex dc7016e94..7bfcc4170 100644--- a/apps/api/src/services/alexandria/README.md+++ b/apps/api/src/services/alexandria/README.md@@ -76,9 +76,8 @@ seven days:An unsettled confirm returns the answer, writes no ledger row, and sends noExchange confirmation; the run stays pending on the Exchange for-reconciliation. A ledger commit or enqueue that fails is refunded at Autumn on-the direct route; on the firebill route the durable charge stands pending-reconciliation. Provider tools do not support forced zero data retention;+reconciliation. A ledger commit or enqueue that fails leaves the durable+Autumn charge standing, pending reconciliation. Provider tools do not support forced zero data retention;those teams are refused before any record is written.## Runtimediff --git a/apps/api/src/services/alexandria/retrieve.test.ts b/apps/api/src/services/alexandria/retrieve.test.tsindex 3411e670f..a061446bc 100644--- a/apps/api/src/services/alexandria/retrieve.test.ts+++ b/apps/api/src/services/alexandria/retrieve.test.ts@@ -44,7 +44,6 @@ vi.mock("../autumn/autumn.service", () => ({hostedMcpFeatureId: async () => "MCP_CREDITS",finalizeCreditsLock: mocks.finalize,refundCredits: mocks.refund,- isRoutedThroughFirebill: async () => false,},featureIdForBillingEndpoint: () => "credits",}));@@ -129,7 +128,7 @@ it("quotes, reserves, executes within budget, settles actual usage, records onceexpect.objectContaining({credits: 3,org_id: "org",- autumnTrackInRequest: true,+ autumnTrackInRequest: false,}),expect.objectContaining({jobId: expect.stringMatching(/^alexandria-bill-/),@@ -197,9 +196,7 @@ it("holds a hosted MCP call MCP credits refuse on the team's own credits", async}),expect.anything(),);- expect(mocks.refund).toHaveBeenCalledExactlyOnceWith(- expect.objectContaining({ value: 3, featureId: "credits" }),- );+ expect(mocks.refund).not.toHaveBeenCalled();});it("reports billing unavailable when the team-credit fallback hold can't be placed", async () => {@@ -332,24 +329,14 @@ it("returns the answer but records nothing when the settle does not land", asyncexpect(mocks.report).not.toHaveBeenCalled();});-it("refunds the direct-Autumn charge and leaves the Exchange usage pending when the ledger enqueue fails", async () => {+it("keeps the charge and leaves the Exchange usage pending when the ledger enqueue fails", async () => {mocks.billAdd.mockRejectedValue(new Error("queue down"));expect((await run()).status).toBe(200);expect(mocks.billAdd).toHaveBeenCalledTimes(3);- expect(mocks.refund).toHaveBeenCalledWith(- expect.objectContaining({ teamId: "team", orgId: "org", value: 3 }),- );+ expect(mocks.refund).not.toHaveBeenCalled();expect(mocks.report).not.toHaveBeenCalled();});-it("refunds a hosted MCP charge to MCP credits when the ledger enqueue fails", async () => {- mocks.billAdd.mockRejectedValue(new Error("queue down"));- expect((await run({ hostedMcp: true })).status).toBe(200);- expect(mocks.refund).toHaveBeenCalledExactlyOnceWith(- expect.objectContaining({ value: 3, featureId: "MCP_CREDITS" }),- );-});-it("treats a team with no org as a skipped hold and executes nothing", async () => {const refused = await run({ orgId: null });expect(refused.status).toBe(503);@@ -793,10 +780,15 @@ it("records nothing on the agent run when nothing was charged", async () => {mocks.store.clear();mocks.finalize.mockResolvedValueOnce(false);await run({ agentRequestId: "agent-run" });- mocks.store.clear();+ expect(mocks.recordCredits).not.toHaveBeenCalled();+});++it("records an agent call's charge when the ledger enqueue fails, since the charge stands", async () => {mocks.billAdd.mockRejectedValue(new Error("queue down"));await run({ agentRequestId: "agent-run" });- expect(mocks.recordCredits).not.toHaveBeenCalled();+ expect(mocks.recordCredits).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({ requestId: "agent-run", credits: 3 }),+ );});it("still answers when the agent run's credits cannot be recorded", async () => {diff --git a/apps/api/src/services/alexandria/retrieve.ts b/apps/api/src/services/alexandria/retrieve.tsindex f4cbbee02..7cca9e577 100644--- a/apps/api/src/services/alexandria/retrieve.ts+++ b/apps/api/src/services/alexandria/retrieve.ts@@ -282,7 +282,6 @@ export async function retrieveProviders(input: {let lockId: string | undefined;let maximumCredits: number;- let refundable = false;const finalize = async (credits: number) => {if (!lockId) return true;const settled = await autumnService.finalizeCreditsLock({@@ -386,10 +385,6 @@ export async function retrieveProviders(input: {// service already answered when it could not name one.let hold: LockCreditsResult = { status: "skipped" };if (input.orgId !== null) {- refundable = !(await autumnService.isRoutedThroughFirebill(- input.teamId,- input.orgId,- ));const orgId = input.orgId;const lock = (feature: string, lockId: string) =>autumnService.lockCredits({@@ -528,12 +523,10 @@ export async function retrieveProviders(input: {input.orgId,input.apiKeyId,credits,- refundable,hostedMcp,): settled;- const charged =- settled && billable && credits > 0 && (recorded || !refundable);+ const charged = settled && billable && credits > 0;if (charged && input.agentRequestId)await recordAgentCredits(input.agentRequestId, id, credits);if (!billable || recorded)@@ -592,7 +585,6 @@ async function recordLedgerUsage(orgId: string | null,apiKeyId: number | null,credits: number,- refundable: boolean,hostedMcp: boolean,): Promise<boolean> {for (let attempt = 1; ; attempt++) {@@ -612,7 +604,7 @@ async function recordLedgerUsage(timestamp: new Date().toISOString(),originating_job_id: id,api_key_id: apiKeyId,- autumnTrackInRequest: refundable,+ autumnTrackInRequest: false,},{ jobId: `alexandria-bill-${id}`, priority: 10 },);@@ -623,31 +615,9 @@ async function recordLedgerUsage(continue;}logger.error(- refundable- ? "Provider usage could not be queued for the ledger; refunding the Autumn charge and leaving the Exchange usage pending for reconciliation"- : "Provider usage could not be queued for the ledger; the firebill charge stands and the Exchange usage stays pending for reconciliation",+ "Provider usage could not be queued for the ledger; the Autumn charge stands and the Exchange usage stays pending for reconciliation",{ chargeId: id, teamId, credits, error },);- // refundable is only ever set with an org in hand; named again so the- // type says so.- if (refundable && orgId !== null)- await autumnService.refundCredits({- teamId,- orgId,- value: credits,- featureId: hostedMcp- ? await autumnService.hostedMcpFeatureId(- teamId,- featureIdForBillingEndpoint("scrape"),- )- : featureIdForBillingEndpoint("scrape"),- properties: {- source: "alexandria",- endpoint: "scrape",- chargeId: id,- },- idempotencyKey: `fc:refund:scrape:${id}`,- });return false;}}diff --git a/apps/api/src/services/billing/__tests__/credit_billing.test.ts b/apps/api/src/services/billing/__tests__/credit_billing.test.tsindex b6ed0d57e..4684b8a61 100644--- a/apps/api/src/services/billing/__tests__/credit_billing.test.ts+++ b/apps/api/src/services/billing/__tests__/credit_billing.test.ts@@ -1,20 +1,13 @@import { vi } from "vitest";// vi.mock is hoisted; factory-referenced values must be created in vi.hoisted().-const {- withAuth,- queueBillingOperation,- trackCredits,- refundCredits,- isRoutedThroughFirebill,-} = vi.hoisted(() => ({- withAuth: vi.fn((fn: any) => fn),- queueBillingOperation: vi.fn<(args: any[]) => Promise<any>>(),- trackCredits: vi.fn<(args: any) => Promise<boolean>>(),- refundCredits: vi.fn<(args: any) => Promise<void>>(),- isRoutedThroughFirebill:- vi.fn<(teamId: string, orgId: string) => Promise<boolean>>(),-}));+const { withAuth, queueBillingOperation, trackCredits, refundCredits } =+ vi.hoisted(() => ({+ withAuth: vi.fn((fn: any) => fn),+ queueBillingOperation: vi.fn<(args: any[]) => Promise<any>>(),+ trackCredits: vi.fn<(args: any) => Promise<boolean>>(),+ refundCredits: vi.fn<(args: any) => Promise<void>>(),+ }));vi.mock("../../../lib/withAuth", () => ({withAuth,@@ -28,7 +21,6 @@ vi.mock("../../autumn/autumn.service", () => ({autumnService: {trackCredits,refundCredits,- isRoutedThroughFirebill,hostedMcpFeatureId: async () => "MCP_CREDITS",},featureIdForBillingEndpoint: (endpoint?: string) =>@@ -57,7 +49,6 @@ beforeEach(() => {queueBillingOperation.mockResolvedValue({ success: true });trackCredits.mockResolvedValue(true);refundCredits.mockResolvedValue(undefined);- isRoutedThroughFirebill.mockResolvedValue(false);});describe("billTeam", () => {@@ -101,8 +92,7 @@ describe("billTeam", () => {);});- it("skips the compensating refund on the firebill route β the charge stands", async () => {- isRoutedThroughFirebill.mockResolvedValue(true);+ it("skips the compensating refund when queueing fails β the charge stands", async () => {queueBillingOperation.mockResolvedValueOnce({success: false,message: "enqueue failed",@@ -118,28 +108,6 @@ describe("billTeam", () => {expect(refundCredits).not.toHaveBeenCalled();});- it("gives the compensating refund its own fc:refund key", async () => {- queueBillingOperation.mockResolvedValueOnce({- success: false,- message: "enqueue failed",- });- trackCredits.mockResolvedValueOnce(true);-- await billTeam("team-1", "org-1", 3, 123, {- endpoint: "map",- jobId: "map-1",- chargeId: "map-1",- externalRequestId: "partner-op-42",- });-- expect(refundCredits).toHaveBeenCalledWith(- expect.objectContaining({- idempotencyKey: "fc:refund:map:map-1",- externalRequestId: "partner-op-42",- }),- );- });-it("marks billing as already tracked when request tracking succeeds", async () => {await billTeam("team-1", "org-1", 3, 123, {endpoint: "search",@@ -169,28 +137,6 @@ describe("billTeam", () => {});});- it("refunds Autumn when queueing fails after request tracking", async () => {- queueBillingOperation.mockResolvedValueOnce({ success: false });-- await billTeam("team-1", "org-1", 3, 123, {- endpoint: "search",- jobId: "job-1",- });-- expect(refundCredits).toHaveBeenCalledWith({- teamId: "team-1",- orgId: "org-1",- value: 3,- properties: {- source: "billTeam",- endpoint: "search",- jobId: "job-1",- apiKeyId: 123,- },- featureId: "SEARCH_CREDITS",- });- });-// preview/keyless teams have no org, and this is the one boundary that// accepts that: the ledger is still enqueued, Autumn is simply not told.it("skips Autumn entirely when the team has no org", async () => {@@ -261,18 +207,4 @@ describe("billTeam for a hosted MCP call", () => {true,]);});-- it("refunds a failed enqueue whole to MCP credits", async () => {- queueBillingOperation.mockResolvedValueOnce({ success: false });-- await billTeam("team-1", "org-1", 3, 123, hostedMcpScrape);-- expect(refundCredits).toHaveBeenCalledExactlyOnceWith(- expect.objectContaining({- value: 3,- featureId: "MCP_CREDITS",- idempotencyKey: "fc:refund:scrape:job-1",- }),- );- });});diff --git a/apps/api/src/services/billing/credit_billing.ts b/apps/api/src/services/billing/credit_billing.tsindex 28d54536d..4c159e082 100644--- a/apps/api/src/services/billing/credit_billing.ts+++ b/apps/api/src/services/billing/credit_billing.ts@@ -78,32 +78,17 @@ export async function billTeam(trackedInRequest,);- // A track only happens with an org in hand; named again so the type says so.- if (!result.success && trackedInRequest && org_id !== null) {- if (await autumnService.isRoutedThroughFirebill(team_id, org_id)) {- // No compensating refund on the firebill route: the tracked charge- // is durable and correct, and a refund here poisons a retried- // request β its track would be deduped by Autumn against the same- // idempotency key (no new charge) while the ledger enqueue succeeds,- // leaving Autumn net-zero for billed work.- logger?.warn(- "billing enqueue failed on the firebill route; charge stands",- { team_id, credits, billing },- );- } else {- await autumnService.refundCredits({- teamId: team_id,- orgId: org_id,- value: credits,- properties: autumnProperties,- featureId,- // Distinct from the track key: the refund is its own charge event.- idempotencyKey: billing.chargeId- ? `fc:refund:${billing.endpoint}:${billing.chargeId}`- : undefined,- externalRequestId: billing.externalRequestId ?? undefined,- });- }+ if (!result.success && trackedInRequest) {+ // No compensating refund: the tracked charge is durable and correct,+ // and a refund here poisons a retried request β its track would be+ // deduped by Autumn against the same idempotency key (no new charge)+ // while the ledger enqueue succeeds, leaving Autumn net-zero for+ // billed work.+ logger?.error("billing enqueue failed after tracking; charge stands", {+ team_id,+ credits,+ billing,+ });}return result;diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex ede5442e9..e5ad3f7f9 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -208,12 +208,7 @@ async function billScrapeJob(job.data.team_id,);- // Resolved outside the try so the catch's refund decision can see it.- let routedToFirebill = false;try {- routedToFirebill = orgId- ? await autumnService.isRoutedThroughFirebill(job.data.team_id, orgId)- : false;trackedInRequest = orgId? await autumnService.trackCredits({teamId: job.data.team_id,@@ -225,20 +220,19 @@ async function billScrapeJob(// (a crawl id is shared by every page β keying on it would collapse// a crawl's pages into one billed event) AND survives a stall// requeue, which re-runs the job under the same id: with this key,- // the re-run dedupes instead of double-billing (firebill route).+ // the re-run dedupes instead of double-billing.idempotencyKey: `fc:track:${billing.endpoint}:${job.id}`,// The caller's own operation id, carried on the charge so// firebill reports it without a request lookup.externalRequestId: billing.externalRequestId ?? undefined,}): false;- // On the firebill route the ledger enqueue must be idempotent by the- // originating job: a stalled job reruns under the same job.id, the- // track dedupes in firebill, and a fresh random billing job id would- // debit the ledger twice (the same pattern monitors already use with- // their deterministic monitor-bill-{id}). Off the route, behavior is- // unchanged.- const billingJobId = routedToFirebill ? `bill-${job.id}` : uuidv7();+ // The ledger enqueue must be idempotent by the originating job: a+ // stalled job reruns under the same job.id, the track dedupes in+ // firebill, and a fresh random billing job id would debit the ledger+ // twice (the same pattern monitors already use with their+ // deterministic monitor-bill-{id}).+ const billingJobId = `bill-${job.id}`;logger.debug(`Adding billing job to queue for team ${job.data.team_id}`,{@@ -254,14 +248,12 @@ async function billScrapeJob(// debit actually commits. A failed commit leaves the event pending// for reconciliation - it is never voided on an ambiguous outcome.//- // On the firebill route the enqueue is retried a few times before- // giving up: the Autumn charge is durable and will not be refunded- // (see the catch below), so a dropped enqueue would leave the ledger- // permanently un-debited. Retries are safe there BECAUSE the job id- // is deterministic β a duplicate add dedupes in BullMQ. Off the- // route the id is random, so it keeps today's single attempt (the- // compensating refund covers it).- const enqueueAttempts = routedToFirebill ? 3 : 1;+ // The enqueue is retried a few times before giving up: the Autumn+ // charge is durable and will not be refunded (see the catch below),+ // so a dropped enqueue would leave the ledger permanently+ // un-debited. Retries are safe BECAUSE the job id is deterministic β+ // a duplicate add dedupes in BullMQ.+ const enqueueAttempts = 3;for (let attempt = 1; ; attempt++) {try {await getBillingQueue().add(@@ -303,39 +295,25 @@ async function billScrapeJob(`Failed to add billing job to queue for team ${job.data.team_id} for ${creditsToBeBilled} credits`,{ error },);- if (trackedInRequest && creditsToBeBilled !== null) {- if (routedToFirebill) {- // No compensating refund on the firebill route: the tracked charge- // is durable and correct (the scrape ran), and refunding it here- // poisons any rerun β its track would dedupe against the intent- // row (no new charge) while the enqueue succeeds, leaving Autumn- // net-zero for work the ledger billed. Reaching this line means- // the bounded enqueue retries above were exhausted: the ledger is- // un-debited for this charge until reconciliation (or a stall- // rerun's deterministic bill-{job.id} enqueue) repairs it β hence- // error level, with everything needed to replay by hand.- logger.error(- "billing enqueue failed after retries on the firebill route; charge stands, ledger un-debited pending reconciliation",- {- jobId: job.id,- teamId: job.data.team_id,- credits: creditsToBeBilled,- billing,- },- );- } else if (orgId) {- await autumnService.refundCredits({+ if (trackedInRequest) {+ // No compensating refund: the tracked charge is durable and correct+ // (the scrape ran), and refunding it here poisons any rerun β its+ // track would dedupe against the intent row (no new charge) while+ // the enqueue succeeds, leaving Autumn net-zero for work the ledger+ // billed. Reaching this line means the bounded enqueue retries above+ // were exhausted: the ledger is un-debited for this charge until+ // reconciliation (or a stall rerun's deterministic bill-{job.id}+ // enqueue) repairs it β hence error level, with everything needed+ // to replay by hand.+ logger.error(+ "billing enqueue failed after retries; charge stands, ledger un-debited pending reconciliation",+ {+ jobId: job.id,teamId: job.data.team_id,- orgId,- value: creditsToBeBilled,- properties: autumnProperties,- featureId,- // Distinct from the track key: a refund is its own charge event- // (same key would 409 as a duplicate of the track and be dropped).- idempotencyKey: `fc:refund:${billing.endpoint}:${job.id}`,- externalRequestId: billing.externalRequestId ?? undefined,- });- }+ credits: creditsToBeBilled,+ billing,+ },+ );}// The billing operation never reached the queue, so no debit will// commit and no confirmation will ever arrive: void the Exchange