Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Await and use team flags in credit estimation
Developers Digest3a434e50b80f
diff --git a/apps/api/src/services/monitoring/runner.test.ts b/apps/api/src/services/monitoring/runner.test.tsindex c728ea89a..2a1c87fd8 100644--- a/apps/api/src/services/monitoring/runner.test.ts+++ b/apps/api/src/services/monitoring/runner.test.ts@@ -10,23 +10,37 @@ import {describe("monitoring runner", () => {describe("estimateActualCredits", () => {- it("prefers scrape-reported credits when present", () => {- expect(estimateActualCredits({ metadata: { creditsUsed: 9 } }, {})).toBe(- 9,- );+ it("prefers scrape-reported credits when present", async () => {+ await expect(+ estimateActualCredits({+ doc: { metadata: { creditsUsed: 9 } },+ options: {},+ internalOptions: { teamFlags: null },+ teamId: "team-1",+ }),+ ).resolves.toBe(9);});- it("bills parsed PDFs by extracted page count", () => {- expect(estimateActualCredits({ metadata: { numPages: 4 } }, {})).toBe(4);+ it("bills parsed PDFs by extracted page count", async () => {+ await expect(+ estimateActualCredits({+ doc: { metadata: { numPages: 4 } },+ options: {},+ internalOptions: { teamFlags: null },+ teamId: "team-1",+ }),+ ).resolves.toBe(4);});- it("adds PDF page credits on top of JSON scrape credits", () => {- expect(- estimateActualCredits(- { metadata: { numPages: 3 } },- { formats: ["json"] },- ),- ).toBe(7);+ it("adds PDF page credits on top of JSON scrape credits", async () => {+ await expect(+ estimateActualCredits({+ doc: { metadata: { numPages: 3 } },+ options: { formats: [{ type: "json" }] },+ internalOptions: { teamFlags: null },+ teamId: "team-1",+ }),+ ).resolves.toBe(7);});});diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 3efdf6842..3f9b0ef7b 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -226,12 +226,17 @@ export async function estimateActualCredits(params: {return doc.metadata.creditsUsed;}+ const flags =+ internalOptions && "teamFlags" in internalOptions+ ? internalOptions.teamFlags+ : ((await getACUCTeam(teamId))?.flags ?? null);+return await calculateCreditsToBeBilled(options,internalOptions,doc,new CostTracking(),- internalOptions?.teamFlags ?? (await getACUCTeam(teamId))?.flags ?? null,+ flags,);}@@ -491,6 +496,7 @@ async function runCrawlTarget(params: {saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,zeroDataRetention: false,bypassBilling: true,+ teamFlags: (await getACUCTeam(params.monitor.team_id))?.flags ?? null,},team_id: params.monitor.team_id,createdAt: Date.now(),@@ -562,7 +568,12 @@ async function runCrawlTarget(params: {if (!doc) continue;const url = getDocumentUrl(doc, (job.data as any)?.url ?? body.url);seen.add(hashMonitorUrl(url));- const pageCredits = estimateActualCredits(doc, body.scrapeOptions);+ const pageCredits = await estimateActualCredits({+ doc,+ options: body.scrapeOptions,+ internalOptions: sc.internalOptions,+ teamId: params.monitor.team_id,+ });credits += pageCredits;pages.push(await diffAndPersistPage({