Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Address review round: append-batch keys, v1 extract key, enqueue retries
Rakshith Ramprakash3363d277a4fc
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 4ea25a619..2010b12fc 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -214,7 +214,14 @@ export async function batchScrapeController(req.auth.team_id,threatScanCredits,req.acuc?.api_key_id ?? null,- { endpoint: "batch_scrape", jobId: id, chargeId: `${id}:threat` },+ {+ endpoint: "batch_scrape",+ jobId: id,+ // Appends reuse the batch id but each append's threat scans are a+ // fresh charge β a shared key would underbill them. Appends stay+ // keyless (per-request UUID in firebill).+ ...(req.body.appendToId ? {} : { chargeId: `${id}:threat` }),+ },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex d7b9ff20e..a0377bf6b 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -87,8 +87,9 @@ export async function crawlController(req.auth.team_id,threatScanCredits,req.acuc?.api_key_id ?? null,- // No chargeId: the crawl id is not minted on this rejection path,- // and there is no other per-charge identity here.+ // No chargeId: a fresh crawl id is minted per request and the+ // rejected crawl is never persisted or queued, so there is no+ // stable per-charge identity that could dedupe a retry.{ endpoint: "crawl" },).catch(error => {_logger.error(diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 20eede498..a1302d0e8 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -192,8 +192,12 @@ export async function extractController(req.auth.team_id,threatScanCredits,req.acuc?.api_key_id ?? null,- // No chargeId: v1 has no extract id in scope on this path.- { endpoint: "extract" },+ {+ endpoint: "extract",+ // Suffixed: the extract's MAIN charge (fire-0) uses the bare+ // extractId β a shared key would collapse the two into one charge.+ chargeId: `${extractId}:threat`,+ },).catch(error => {_logger.error(`Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex b01e0745b..862280b27 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -242,7 +242,13 @@ export async function batchScrapeController(req.auth.team_id,threatScanCredits,req.acuc?.api_key_id ?? null,- { ...billing, chargeId: `${id}:threat` },+ {+ ...billing,+ // Appends reuse the batch id but each append's threat scans are a+ // fresh charge β a shared key would underbill them. Appends stay+ // keyless (per-request UUID in firebill).+ ...(req.body.appendToId ? {} : { chargeId: `${id}:threat` }),+ },).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for ${threatScanCredits} threat scan credit(s): ${error}`,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b5ef6f383..cc3e34ad2 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -91,8 +91,9 @@ export async function crawlController(req.auth.team_id,threatScanCredits,req.acuc?.api_key_id ?? null,- // No chargeId: the crawl id is not minted on this rejection path,- // and there is no other per-charge identity here.+ // No chargeId: a fresh crawl id is minted per request and the+ // rejected crawl is never persisted or queued, so there is no+ // stable per-charge identity that could dedupe a retry.{ endpoint: "crawl" },).catch(error => {_logger.error(diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 04dd190b4..323f6575b 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -209,26 +209,48 @@ async function billScrapeJob(// the rest, including confirming the Exchange access event once the// debit actually commits. A failed commit leaves the event pending// for reconciliation - it is never voided on an ambiguous outcome.- await getBillingQueue().add(- "bill_team",- {- team_id: job.data.team_id,- credits: creditsToBeBilled,- billing,- is_extract: false,- timestamp: new Date().toISOString(),- originating_job_id: job.id,- api_key_id: job.data.apiKeyId,- autumnTrackInRequest: trackedInRequest,- ...(exchange?.accessEventId === undefined- ? {}- : { exchangeAccessEventId: exchange.accessEventId }),- },- {- jobId: billingJobId,- priority: 10,- },- );+ //+ // On the firebill route the enqueue is retried a few times before+ // giving up: the Autumn charge is durable and will not be refunded+ // (see the catch below), so a dropped enqueue would leave the ledger+ // permanently un-debited. Retries are safe there BECAUSE the job id+ // is deterministic β a duplicate add dedupes in BullMQ. Off the+ // route the id is random, so it keeps today's single attempt (the+ // compensating refund covers it).+ const enqueueAttempts = routedToFirebill ? 3 : 1;+ for (let attempt = 1; ; attempt++) {+ try {+ await getBillingQueue().add(+ "bill_team",+ {+ team_id: job.data.team_id,+ credits: creditsToBeBilled,+ billing,+ is_extract: false,+ timestamp: new Date().toISOString(),+ originating_job_id: job.id,+ api_key_id: job.data.apiKeyId,+ autumnTrackInRequest: trackedInRequest,+ ...(exchange?.accessEventId === undefined+ ? {}+ : { exchangeAccessEventId: exchange.accessEventId }),+ },+ {+ jobId: billingJobId,+ priority: 10,+ },+ );+ break;+ } catch (enqueueError) {+ if (attempt >= enqueueAttempts) throw enqueueError;+ logger.warn("billing enqueue failed; retrying", {+ attempt,+ billingJobId,+ error: enqueueError,+ });+ await new Promise(resolve => setTimeout(resolve, 250 * attempt));+ }+ }return creditsToBeBilled;} catch (error) {@@ -240,13 +262,21 @@ async function billScrapeJob(if (routedToFirebill) {// No compensating refund on the firebill route: the tracked charge// is durable and correct (the scrape ran), and refunding it here- // poisons a rerun β the rerun's track dedupes against the intent- // row (no new charge) while the ledger enqueue succeeds, leaving- // Autumn net-zero for work the ledger billed. The rerun retries- // the enqueue under the deterministic bill-{job.id} instead.- logger.warn(- "billing enqueue failed on the firebill route; charge stands, enqueue retries on rerun",- { jobId: job.id, credits: creditsToBeBilled },+ // poisons any rerun β its track would dedupe against the intent+ // row (no new charge) while the enqueue succeeds, leaving Autumn+ // net-zero for work the ledger billed. Reaching this line means+ // the bounded enqueue retries above were exhausted: the ledger is+ // un-debited for this charge until reconciliation (or a stall+ // rerun's deterministic bill-{job.id} enqueue) repairs it β hence+ // error level, with everything needed to replay by hand.+ logger.error(+ "billing enqueue failed after retries on the firebill route; charge stands, ledger un-debited pending reconciliation",+ {+ jobId: job.id,+ teamId: job.data.team_id,+ credits: creditsToBeBilled,+ billing,+ },);} else {await autumnService.refundCredits({