Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Update extract.ts

Nicolasa23364e5dae6
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index ebe248bae..9390ab9a1 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -7,180 +7,16 @@ import {
ExtractResponse,
MapDocument,
} from "./types";
-import { v4 as uuidv4 } from "uuid";
-import { getJobPriority } from "../../lib/job-priority";
-import { PlanType } from "../../types";
-import { getMapResults } from "./map";
-
-
export async function extractController(
req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>,
res: Response<ExtractResponse>
) {
req.body = extractRequestSchema.parse(req.body);
- let earlyReturn = false;
-
- const origin = req.body.origin;
- const timeout = req.body.timeout;
-// const pageOptions = legacyScrapeOptions(req.body);
-// const extractorOptions = req.body.extract ? legacyExtractorOptions(req.body.extract) : undefined;
- const jobId = uuidv4();
-
- const startTime = new Date().getTime();
- const jobPriority = await getJobPriority({
- plan: req.auth.plan as PlanType,
- team_id: req.auth.team_id,
- basePriority: 10,
- });
-
- const urls = req.body.urls;
- const mappedDocuments: MapDocument[] = [];
-
- const prompt = req.body.prompt;
- // const keywords = await generateBasicCompletion(`If the user's prompt is: "${prompt}", what are the most important keywords besides the extraction task? Output only the keywords, separated by commas.`);
-
- for (const url of urls) {
- if (url.endsWith("/*")) {
- const mapResults = await getMapResults({
- url: url.slice(0, -2),
- search: req.body.prompt,
- limit: 100,
- ignoreSitemap: true,
- includeSubdomains: false,
- crawlerOptions: {},
- teamId: req.auth.team_id,
- plan: req.auth.plan,
- origin: req.body.origin,
- subId: req.acuc?.sub_id,
- includeMetadata: true
- });
- // // top 3 links
- // const top3Links = (mapResults.links as MapDocument[]).slice(0, 3);
- // console.log(top3Links);
- // // console.log(top3Links);
- // mappedDocuments.push(...(mapResults.links as MapDocument[]));
- // // transform mappedUrls to just documents
- // // we quickly rerank
- // const rerank = await rerankDocuments(mappedDocuments.map(x => `URL: ${x.url}\nTITLE: ${x.title}\nDESCRIPTION: ${x.description}`), "What URLs are most relevant to the following prompt: " + (req.body.prompt || '').toLocaleLowerCase().replace("extract", " ").replace("extract ", " "));
- // console.log(rerank);
- } else {
- mappedDocuments.push({ url });
- }
- }
-
- req.body.urls = mappedDocuments.map(x => x.url);
-
-
-
-// const job = await addScrapeJob(
-// {
-// url: req.body.url,
-// mode: "single_urls",
-// crawlerOptions: {},
-// team_id: req.auth.team_id,
-// plan: req.auth.plan,
-// pageOptions,
-// extractorOptions,
-// origin: req.body.origin,
-// is_scrape: true,
-// },
-// {},
-// jobId,
-// jobPriority
-// );
-
-// const totalWait = (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds : 0) + a, 0);
-
-// let doc: any | undefined;
-// try {
-// doc = (await waitForJob(job.id, timeout + totalWait))[0];
-// } catch (e) {
-// Logger.error(`Error in scrapeController: ${e}`);
-// if (e instanceof Error && e.message.startsWith("Job wait")) {
-// return res.status(408).json({
-// success: false,
-// error: "Request timed out",
-// });
-// } else {
-// return res.status(500).json({
-// success: false,
-// error: `(Internal server error) - ${e && e?.message ? e.message : e} ${
-// extractorOptions && extractorOptions.mode !== "markdown"
-// ? " - Could be due to LLM parsing issues"
-// : ""
-// }`,
-// });
-// }
-// }
-
-// await job.remove();
-
-// if (!doc) {
-// console.error("!!! PANIC DOC IS", doc, job);
-// return res.status(200).json({
-// success: true,
-// warning: "No page found",
-// data: doc,
-// });
-// }
-
-// delete doc.index;
-// delete doc.provider;
-
-// const endTime = new Date().getTime();
-// const timeTakenInSeconds = (endTime - startTime) / 1000;
-// const numTokens =
-// doc && doc.markdown
-// ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
-// : 0;
-
-// let creditsToBeBilled = 1; // Assuming 1 credit per document
-// if (earlyReturn) {
-// // Don't bill if we're early returning
-// return;
-// }
-// if(req.body.extract && req.body.formats.includes("extract")) {
-// creditsToBeBilled = 5;
-// }
-
-// billTeam(req.auth.team_id, req.acuc?.sub_id, creditsToBeBilled).catch(error => {
-// Logger.error(`Failed to bill team ${req.auth.team_id} for ${creditsToBeBilled} credits: ${error}`);
-// // Optionally, you could notify an admin or add to a retry queue here
-// });
-
-// if (!pageOptions || !pageOptions.includeRawHtml) {
-// if (doc && doc.rawHtml) {
-// delete doc.rawHtml;
-// }
-// }
-
-// if(pageOptions && pageOptions.includeExtract) {
-// if(!pageOptions.includeMarkdown && doc && doc.markdown) {
-// delete doc.markdown;
-// }
-// }
-
-// logJob({
-// job_id: jobId,
-// success: true,
-// message: "Scrape completed",
-// num_docs: 1,
-// docs: [doc],
-// time_taken: timeTakenInSeconds,
-// team_id: req.auth.team_id,
-// mode: "scrape",
-// url: req.body.url,
-// crawlerOptions: {},
-// pageOptions: pageOptions,
-// origin: origin,
-// extractor_options: extractorOptions,
-// num_tokens: numTokens,
-// });
return res.status(200).json({
success: true,
data: {} as Document,
- scrape_id: origin?.includes("website") ? jobId : undefined,
+ scrape_id: undefined,
});
}