Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/v0/scrape): populate credits_billed field (#2066)
Gergő Móricz6b39d1263edc
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex f1e0aa735..cd925145a 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -1,19 +1,16 @@import { ExtractorOptions, PageOptions } from "./../../lib/entities";import { Request, Response } from "express";import {- billTeam,checkTeamCredits,} from "../../services/billing/credit_billing";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../types";import {- fromLegacyCombo,TeamFlags,toLegacyDocument,url as urlSchema,} from "../v1/types";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; // Import the isUrlBlocked function-import { numTokensFromString } from "../../lib/LLM-extraction/helpers";import {defaultPageOptions,defaultExtractorOptions,@@ -27,15 +24,13 @@ import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";import * as Sentry from "@sentry/node";import { getJobPriority } from "../../lib/job-priority";-import { fromLegacyScrapeOptions } from "../v1/types";import { ZodError } from "zod";import { Document as V0Document } from "./../../lib/entities";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";-import { getJobFromGCS } from "../../lib/gcs-jobs";import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";-export async function scrapeHelper(+async function scrapeHelper(jobId: string,req: Request,team_id: string,@@ -85,7 +80,6 @@ export async function scrapeHelper(internalOptions,origin: req.body.origin ?? defaultOrigin,integration: req.body.integration,- is_scrape: true,startTime: Date.now(),zeroDataRetention: false, // not supported on v0apiKeyId,@@ -234,7 +228,6 @@ export async function scrapeController(req: Request, res: Response) {});}- const startTime = new Date().getTime();const result = await scrapeHelper(jobId,req,@@ -246,44 +239,6 @@ export async function scrapeController(req: Request, res: Response) {chunk?.flags ?? null,chunk?.api_key_id ?? null,);- const endTime = new Date().getTime();- const timeTakenInSeconds = (endTime - startTime) / 1000;- const numTokens =- result.data && (result.data as V0Document).markdown- ? numTokensFromString(- (result.data as V0Document).markdown!,- "gpt-3.5-turbo",- )- : 0;-- if (result.success) {- let creditsToBeBilled = 1;- const creditsPerLLMExtract = 4;-- if (extractorOptions.mode.includes("llm-extraction")) {- // creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length);- creditsToBeBilled += creditsPerLLMExtract;- }-- let startTimeBilling = new Date().getTime();-- if (earlyReturn) {- // Don't bill if we're early returning- return;- }- if (creditsToBeBilled > 0) {- // billing for doc done on queue end, bill only for llm extraction- billTeam(team_id, chunk?.sub_id, creditsToBeBilled, chunk?.api_key_id ?? null, logger).catch(- (error) => {- logger.error(- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,- { error },- );- // Optionally, you could notify an admin or add to a retry queue here- },- );- }- }let doc = result.data;if (!pageOptions || !pageOptions.includeRawHtml) {