Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick:
Nicolasa4f15260a7fe
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 4ac32dddf..ebe248bae 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -10,8 +10,6 @@ import {import { v4 as uuidv4 } from "uuid";import { getJobPriority } from "../../lib/job-priority";import { PlanType } from "../../types";-import { rerankDocuments } from "../../lib/extract/reranker";-import { generateBasicCompletion } from "../../lib/extract/completions";import { getMapResults } from "./map";@@ -40,7 +38,7 @@ export async function extractController(const mappedDocuments: MapDocument[] = [];const prompt = req.body.prompt;- const keywords = await generateBasicCompletion(`If the user's prompt is: "${prompt}", what are the most important keywords besides the extraction task? Output only the keywords, separated by commas.`);+ // const keywords = await generateBasicCompletion(`If the user's prompt is: "${prompt}", what are the most important keywords besides the extraction task? Output only the keywords, separated by commas.`);for (const url of urls) {if (url.endsWith("/*")) {@@ -57,15 +55,15 @@ export async function extractController(subId: req.acuc?.sub_id,includeMetadata: true});- // top 3 links- const top3Links = (mapResults.links as MapDocument[]).slice(0, 3);- console.log(top3Links);- // console.log(top3Links);- mappedDocuments.push(...(mapResults.links as MapDocument[]));- // transform mappedUrls to just documents- // we quickly rerank- const rerank = await rerankDocuments(mappedDocuments.map(x => `URL: ${x.url}\nTITLE: ${x.title}\nDESCRIPTION: ${x.description}`), "What URLs are most relevant to the following prompt: " + (req.body.prompt || '').toLocaleLowerCase().replace("extract", " ").replace("extract ", " "));- console.log(rerank);+ // // top 3 links+ // const top3Links = (mapResults.links as MapDocument[]).slice(0, 3);+ // console.log(top3Links);+ // // console.log(top3Links);+ // mappedDocuments.push(...(mapResults.links as MapDocument[]));+ // // transform mappedUrls to just documents+ // // we quickly rerank+ // const rerank = await rerankDocuments(mappedDocuments.map(x => `URL: ${x.url}\nTITLE: ${x.title}\nDESCRIPTION: ${x.description}`), "What URLs are most relevant to the following prompt: " + (req.body.prompt || '').toLocaleLowerCase().replace("extract", " ").replace("extract ", " "));+ // console.log(rerank);} else {mappedDocuments.push({ url });}diff --git a/apps/api/src/lib/extract/completions.ts b/apps/api/src/lib/extract/completions.tsindex 230584b4a..fa75594bf 100644--- a/apps/api/src/lib/extract/completions.ts+++ b/apps/api/src/lib/extract/completions.ts@@ -82,7 +82,7 @@ export async function generateFinalExtraction({const jsonCompletion = await openai.beta.chat.completions.parse({model,messages: [- { role: "system", content: systemPrompt },+ { role: "system", content: systemPrompt ?? "" },{ role: "user", content: [{ type: "text", text: extractionContent }] },{role: "user",@@ -108,9 +108,8 @@ export async function generateFinalExtraction({}const extraction = jsonCompletion.choices[0].message.parsed;-return {- content: extraction,+ content: extraction ?? "",metadata: {numTokens,warning,diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 3eaace3b2..e6055a99d 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -18,6 +18,7 @@ import { logger } from "../lib/logger";import { scrapeStatusController } from "../controllers/v1/scrape-status";import { concurrencyCheckController } from "../controllers/v1/concurrency-check";import { batchScrapeController } from "../controllers/v1/batch-scrape";+import { extractController } from "../controllers/v1/extract";// import { crawlPreviewController } from "../../src/controllers/v1/crawlPreview";// import { crawlJobStatusPreviewController } from "../../src/controllers/v1/status";// import { searchController } from "../../src/controllers/v1/search";@@ -178,6 +179,14 @@ v1Router.ws(crawlStatusWSController);+v1Router.post(+ "/extract",+ authMiddleware(RateLimiterMode.Scrape),+ checkCreditsMiddleware(1),+ blocklistMiddleware,+ wrap(extractController)+);+// v1Router.post("/crawlWebsitePreview", crawlPreviewController);@@ -199,3 +208,4 @@ v1Router.delete(// Health/Probe routes// v1Router.get("/health/liveness", livenessController);// v1Router.get("/health/readiness", readinessController);+