Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Loggin num tokens

Nicolas3de4997f4dbb
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 3797032a8..96e1ee78a 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -197,51 +197,6 @@ export async function extractController(
// Optionally, you could notify an admin or add to a retry queue here
});
- // if (!req.body.formats.includes("rawHtml")) {
- // if (doc && doc.rawHtml) {
- // delete doc.rawHtml;
- // }
- // }
-
-
-
-
-
- // billTeam(teamId, subId, 1).catch((error) => {
- // logger.error(
- // `Failed to bill team ${teamId} for 1 credit: ${error}`
- // );
- // });
-
- // const linksToReturn = links.slice(0, limit);
-
- // logJob({
- // job_id: id,
- // success: links.length > 0,
- // message: "Extract completed",
- // num_docs: linksToReturn.length,
- // docs: linksToReturn,
- // time_taken: (new Date().getTime() - Date.now()) / 1000,
- // team_id: teamId,
- // mode: "extract",
- // url: urls[0],
- // crawlerOptions: {},
- // scrapeOptions: {},
- // origin: origin ?? "api",
- // num_tokens: 0,
- // });
-
- // return {
-
- // };
-
-
-
- // const response = {
- // success: true as const,
- // data: result.data,
- // scrape_id: result.scrape_id
- // };
console.log("completions.extract", completions.extract);
@@ -264,9 +219,9 @@ export async function extractController(
url: req.body.urls.join(", "),
scrapeOptions: req.body,
origin: req.body.origin ?? "api",
- num_tokens: 0, // TODO: fix
+ num_tokens: completions.numTokens ?? 0
});
-
+
return res.status(200).json({
success: true,
data: data,
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 41d767791..3866683aa 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -58,7 +58,7 @@ function normalizeSchema(x: any): any {
}
}
-export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string): Promise<{ extract: any, warning: string | undefined }> {
+export async function generateOpenAICompletions(logger: Logger, options: ExtractOptions, markdown?: string, previousWarning?: string): Promise<{ extract: any, numTokens: number, warning: string | undefined }> {
let extract: any;
let warning: string | undefined;
@@ -170,7 +170,7 @@ export async function generateOpenAICompletions(logger: Logger, options: Extract
if (options.schema && options.schema.type === "array" && !schema?.required?.includes("items")) {
extract = extract?.items;
}
- return { extract, warning };
+ return { extract, warning, numTokens };
}
export async function performLLMExtract(meta: Meta, document: Document): Promise<Document> {