Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update extract.ts
Nicolas5522d6af7d2c
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex d462e651d..e5f2595c8 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -103,7 +103,6 @@ export async function extractController(// If we don't have enough high-quality links, try with lower thresholdif (filteredLinks.length < MIN_REQUIRED_LINKS) {- console.log(`Only found ${filteredLinks.length} links with score > ${INITIAL_SCORE_THRESHOLD}. Trying lower threshold...`);logger.info(`Only found ${filteredLinks.length} links with score > ${INITIAL_SCORE_THRESHOLD}. Trying lower threshold...`);filteredLinks = filterAndProcessLinks(mappedLinks, linksAndScores, FALLBACK_SCORE_THRESHOLD);@@ -208,24 +207,17 @@ export async function extractController(logger.child({ method: "extractController/generateOpenAICompletions" }),{mode: "llm",- systemPrompt: "Only use the provided content to answer the question.",+ systemPrompt: "Always prioritize using the provided content to answer the question. Do not make up an answer. Be concise and follow the schema if provided.",prompt: req.body.prompt,schema: req.body.schema,},docs.map(x => buildDocument(x)).join('\n'));- // console.log("completions", completions);-- // if(req.body.extract && req.body.formats.includes("extract")) {- // creditsToBeBilled = 5;- // }-// TODO: change this later// While on beta, we're billing 5 credits per link discovered/scraped.billTeam(req.auth.team_id, req.acuc?.sub_id, links.length * 5).catch(error => {logger.error(`Failed to bill team ${req.auth.team_id} for ${links.length * 5} credits: ${error}`);- // Optionally, you could notify an admin or add to a retry queue here});let data = completions.extract ?? {};@@ -254,6 +246,13 @@ export async function extractController(});}+/**+ * Filters links based on their similarity score to the search query.+ * @param mappedLinks - The list of mapped links to filter.+ * @param linksAndScores - The list of links and their similarity scores.+ * @param threshold - The score threshold to filter by.+ * @returns The filtered list of links.+ */function filterAndProcessLinks(mappedLinks: MapDocument[],linksAndScores: { link: string, linkWithContext: string, score: number, originalIndex: number }[],