Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Loggin num tokens
Nicolas3de4997f4dbb
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 3797032a8..96e1ee78a 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -197,51 +197,6 @@ export async function extractController(// Optionally, you could notify an admin or add to a retry queue here});- // if (!req.body.formats.includes("rawHtml")) {- // if (doc && doc.rawHtml) {- // delete doc.rawHtml;- // }- // }------ // billTeam(teamId, subId, 1).catch((error) => {- // logger.error(- // `Failed to bill team ${teamId} for 1 credit: ${error}`- // );- // });-- // const linksToReturn = links.slice(0, limit);-- // logJob({- // job_id: id,- // success: links.length > 0,- // message: "Extract completed",- // num_docs: linksToReturn.length,- // docs: linksToReturn,- // time_taken: (new Date().getTime() - Date.now()) / 1000,- // team_id: teamId,- // mode: "extract",- // url: urls[0],- // crawlerOptions: {},- // scrapeOptions: {},- // origin: origin ?? "api",- // num_tokens: 0,- // });-- // return {-- // };---- // const response = {- // success: true as const,- // data: result.data,- // scrape_id: result.scrape_id- // };console.log("completions.extract", completions.extract);@@ -264,9 +219,9 @@ export async function extractController(url: req.body.urls.join(", "),scrapeOptions: req.body,origin: req.body.origin ?? "api",- num_tokens: 0, // TODO: fix+ num_tokens: completions.numTokens ?? 0});-+return res.status(200).json({success: true,data: data,diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 41d767791..3866683aa 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -58,7 +58,7 @@ function normalizeSchema(x: any): any {}}-export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string): Promise<{ extract: any, warning: string | undefined }> {+export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string): Promise<{ extract: any, numTokens: number, warning: string | undefined }> {let extract: any;let warning: string | undefined;@@ -170,7 +170,7 @@ export async function generateOpenAICompletions(logger: Logger, options: Extractif (options.schema && options.schema.type === "array" && !schema?.required?.includes("items")) {extract = extract?.items;}- return { extract, warning };+ return { extract, warning, numTokens };}export async function performLLMExtract(meta: Meta, document: Document): Promise<Document> {