Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api/v0/scrape): populate credits_billed field (#2066)

Gergő Móricz6b39d1263edc
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index f1e0aa735..cd925145a 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -1,19 +1,16 @@
import { ExtractorOptions, PageOptions } from "./../../lib/entities";
import { Request, Response } from "express";
import {
- billTeam,
checkTeamCredits,
} from "../../services/billing/credit_billing";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../types";
import {
- fromLegacyCombo,
TeamFlags,
toLegacyDocument,
url as urlSchema,
} from "../v1/types";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist"; // Import the isUrlBlocked function
-import { numTokensFromString } from "../../lib/LLM-extraction/helpers";
import {
defaultPageOptions,
defaultExtractorOptions,
@@ -27,15 +24,13 @@ import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
import * as Sentry from "@sentry/node";
import { getJobPriority } from "../../lib/job-priority";
-import { fromLegacyScrapeOptions } from "../v1/types";
import { ZodError } from "zod";
import { Document as V0Document } from "./../../lib/entities";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
-import { getJobFromGCS } from "../../lib/gcs-jobs";
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
-export async function scrapeHelper(
+async function scrapeHelper(
jobId: string,
req: Request,
team_id: string,
@@ -85,7 +80,6 @@ export async function scrapeHelper(
internalOptions,
origin: req.body.origin ?? defaultOrigin,
integration: req.body.integration,
- is_scrape: true,
startTime: Date.now(),
zeroDataRetention: false, // not supported on v0
apiKeyId,
@@ -234,7 +228,6 @@ export async function scrapeController(req: Request, res: Response) {
});
}
- const startTime = new Date().getTime();
const result = await scrapeHelper(
jobId,
req,
@@ -246,44 +239,6 @@ export async function scrapeController(req: Request, res: Response) {
chunk?.flags ?? null,
chunk?.api_key_id ?? null,
);
- const endTime = new Date().getTime();
- const timeTakenInSeconds = (endTime - startTime) / 1000;
- const numTokens =
- result.data && (result.data as V0Document).markdown
- ? numTokensFromString(
- (result.data as V0Document).markdown!,
- "gpt-3.5-turbo",
- )
- : 0;
-
- if (result.success) {
- let creditsToBeBilled = 1;
- const creditsPerLLMExtract = 4;
-
- if (extractorOptions.mode.includes("llm-extraction")) {
- // creditsToBeBilled = creditsToBeBilled + (creditsPerLLMExtract * filteredDocs.length);
- creditsToBeBilled += creditsPerLLMExtract;
- }
-
- let startTimeBilling = new Date().getTime();
-
- if (earlyReturn) {
- // Don't bill if we're early returning
- return;
- }
- if (creditsToBeBilled > 0) {
- // billing for doc done on queue end, bill only for llm extraction
- billTeam(team_id, chunk?.sub_id, creditsToBeBilled, chunk?.api_key_id ?? null, logger).catch(
- (error) => {
- logger.error(
- `Failed to bill team ${team_id} for ${creditsToBeBilled} credits`,
- { error },
- );
- // Optionally, you could notify an admin or add to a retry queue here
- },
- );
- }
- }
let doc = result.data;
if (!pageOptions || !pageOptions.includeRawHtml) {