Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge remote-tracking branch 'origin/main' into nsc/new-extract
rafaelmmiller36cf49c95980
diff --cc apps/api/src/controllers/v1/map.tsindex ba7be01f2,ab9a5ed7f..8ab5c1354--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@@ -1,11 -1,6 +1,6 @@@import { Response } from "express";import { v4 as uuidv4 } from "uuid";- import {- MapDocument,- mapRequestSchema,- RequestWithAuth,- scrapeOptions,- } from "./types";-import { mapRequestSchema, RequestWithAuth, scrapeOptions } from "./types";++import { MapDocument, mapRequestSchema, RequestWithAuth, scrapeOptions } from "./types";import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";import { MapResponse, MapRequest } from "./types";import { configDotenv } from "dotenv";@@@ -30,46 -25,24 +25,48 @@@ const MAX_MAP_LIMIT = 5000// Max Links that "Smart /map" can returnconst MAX_FIRE_ENGINE_RESULTS = 1000;-export async function mapController(- req: RequestWithAuth<{}, MapResponse, MapRequest>,- res: Response<MapResponse>-) {- const startTime = new Date().getTime();-- req.body = mapRequestSchema.parse(req.body);-- const limit: number = req.body.limit ?? MAX_MAP_LIMIT;+interface MapResult {+ success: boolean;+ links: string[] | any[];+ scrape_id?: string;+ job_id: string;+ time_taken: number;+}+export async function getMapResults({+ url,+ search,+ limit = MAX_MAP_LIMIT,+ ignoreSitemap = false,+ includeSubdomains = true,+ crawlerOptions = {},+ teamId,+ plan,+ origin,+ includeMetadata = false,+ allowExternalLinks+}: {+ url: string;+ search?: string;+ limit?: number;+ ignoreSitemap?: boolean;+ includeSubdomains?: boolean;+ crawlerOptions?: any;+ teamId: string;+ plan?: string;+ origin?: string;+ includeMetadata?: boolean;+ allowExternalLinks?: boolean;+}): Promise<MapResult> {const id = uuidv4();- let links: string[] = [req.body.url];+ let links: string[] = [url];++ let mapResults: MapDocument[] = [];const sc: StoredCrawl = {- originUrl: req.body.url,+ originUrl: url,crawlerOptions: {- ...req.body,- limit: req.body.sitemapOnly ? 10000000 : limit,+ ...crawlerOptions,++ limit: crawlerOptions.sitemapOnly ? 10000000 : limit,scrapeOptions: undefined,},scrapeOptions: scrapeOptions.parse({}),@@@ -81,136 -54,130 +78,161 @@@const crawler = crawlToCrawler(id, sc);- let urlWithoutWww = url.replace("www.", "");-- let mapUrl = search && allowExternalLinks- ? `${search} ${urlWithoutWww}`- : search ? `${search} site:${urlWithoutWww}`- : `site:${url}`;-- const resultsPerPage = 100;- const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);-- const cacheKey = `fireEngineMap:${mapUrl}`;- const cachedResult = null;-- let allResults: any[] = [];- let pagePromises: Promise<any>[] = [];+ // If sitemapOnly is true, only get links from sitemap- if (req.body.sitemapOnly) {++ if (crawlerOptions.sitemapOnly) {++ if (includeMetadata) {++ throw new Error("includeMetadata is not supported with sitemapOnly");++ }+- if (cachedResult) {- allResults = JSON.parse(cachedResult);- } else {- const fetchPage = async (page: number) => {- return fireEngineMap(mapUrl, {- numResults: resultsPerPage,- page: page,+ const sitemap = await crawler.tryGetSitemap(true, true);+ if (sitemap !== null) {+ sitemap.forEach((x) => {+ links.push(x.url);});- };+ links = links.slice(1)+ .map((x) => {+ try {+ return checkAndUpdateURLForMap(x).url.trim();+ } catch (_) {+ return null;+ }+ })+ .filter((x) => x !== null) as string[];+ // links = links.slice(1, limit); // don't slice, unnecessary+ }+ } else {- let urlWithoutWww = req.body.url.replace("www.", "");++ let urlWithoutWww = url.replace("www.", "");- pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1));- allResults = await Promise.all(pagePromises);- let mapUrl = req.body.search- ? `"${req.body.search}" site:${urlWithoutWww}`- : `site:${req.body.url}`;++ let mapUrl = search && allowExternalLinks++ ? `${search} ${urlWithoutWww}`++ : search ? `${search} site:${urlWithoutWww}`++ : `site:${url}`;- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours- }+ const resultsPerPage = 100;- const maxPages = Math.ceil(- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage- );++ const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);- console.log("allResults", allResults);- // Parallelize sitemap fetch with serper search- const [sitemap, ...searchResults] = await Promise.all([- ignoreSitemap ? null : crawler.tryGetSitemap(),- ...(cachedResult ? [] : pagePromises),- ]);+ const cacheKey = `fireEngineMap:${mapUrl}`;+ const cachedResult = null;- if (!cachedResult) {- allResults = searchResults;- }+ let allResults: any[] = [];+ let pagePromises: Promise<any>[] = [];- if (sitemap !== null) {- sitemap.forEach((x) => {- links.push(x.url);- });- }+ if (cachedResult) {+ allResults = JSON.parse(cachedResult);+ } else {+ const fetchPage = async (page: number) => {+ return fireEngineMap(mapUrl, {+ numResults: resultsPerPage,+ page: page,+ });+ };++ pagePromises = Array.from({ length: maxPages }, (_, i) =>+ fetchPage(i + 1)+ );+ allResults = await Promise.all(pagePromises);++ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours+ }- let mapResults : MapDocument[] = allResults- .flat()- .filter((result) => result !== null && result !== undefined);+ // Parallelize sitemap fetch with serper search+ const [sitemap, ...searchResults] = await Promise.all([- req.body.ignoreSitemap ? null : crawler.tryGetSitemap(true),++ ignoreSitemap ? null : crawler.tryGetSitemap(),+ ...(cachedResult ? [] : pagePromises),+ ]);- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);- if (mapResults.length > minumumCutoff) {- mapResults = mapResults.slice(0, minumumCutoff);- }+ if (!cachedResult) {+ allResults = searchResults;+ }- if (mapResults.length > 0) {- if (search) {- // Ensure all map results are first, maintaining their order- links = [- mapResults[0].url,- ...mapResults.slice(1).map((x) => x.url),- ...links,- ];- } else {- mapResults.map((x) => {+ if (sitemap !== null) {+ sitemap.forEach((x) => {links.push(x.url);});}- }- // Perform cosine similarity between the search query and the list of links- if (search) {- const searchQuery = search.toLowerCase();- links = performCosineSimilarity(links, searchQuery);- }- let mapResults = allResults++ mapResults = allResults+ .flat()+ .filter((result) => result !== null && result !== undefined);++ const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);+ if (mapResults.length > minumumCutoff) {+ mapResults = mapResults.slice(0, minumumCutoff);+ }- links = links- .map((x) => {- try {- return checkAndUpdateURLForMap(x).url.trim();- } catch (_) {- return null;+ if (mapResults.length > 0) {- if (req.body.search) {++ if (search) {+ // Ensure all map results are first, maintaining their order+ links = [+ mapResults[0].url,+ ...mapResults.slice(1).map((x) => x.url),+ ...links,+ ];+ } else {+ mapResults.map((x) => {+ links.push(x.url);+ });}- })- .filter((x) => x !== null) as string[];+ }- // allows for subdomains to be included- links = links.filter((x) => isSameDomain(x, url));+ // Perform cosine similarity between the search query and the list of links- if (req.body.search) {- const searchQuery = req.body.search.toLowerCase();-++ if (search) {++ const searchQuery = search.toLowerCase();+ links = performCosineSimilarity(links, searchQuery);+ }- // if includeSubdomains is false, filter out subdomains- if (!includeSubdomains) {- links = links.filter((x) => isSameSubdomain(x, url));- }+ links = links+ .map((x) => {+ try {+ return checkAndUpdateURLForMap(x).url.trim();+ } catch (_) {+ return null;+ }+ })+ .filter((x) => x !== null) as string[];++ // allows for subdomains to be included- links = links.filter((x) => isSameDomain(x, req.body.url));++ links = links.filter((x) => isSameDomain(x, url));++ // if includeSubdomains is false, filter out subdomains- if (!req.body.includeSubdomains) {- links = links.filter((x) => isSameSubdomain(x, req.body.url));++ if (!includeSubdomains) {++ links = links.filter((x) => isSameSubdomain(x, url));+ }- // remove duplicates that could be due to http/https or www- links = removeDuplicateUrls(links);+ // remove duplicates that could be due to http/https or www+ links = removeDuplicateUrls(links);- links.slice(0, limit);+ }+ const linksToReturn = links.slice(0, limit);++ return {+ success: true,+ links: includeMetadata ? mapResults : linksToReturn,+ scrape_id: origin?.includes("website") ? id : undefined,+ job_id: id,+ time_taken: (new Date().getTime() - Date.now()) / 1000,+ };+}++export async function mapController(+ req: RequestWithAuth<{}, MapResponse, MapRequest>,+ res: Response<MapResponse>+) {+ req.body = mapRequestSchema.parse(req.body);++ const result = await getMapResults({+ url: req.body.url,+ search: req.body.search,+ limit: req.body.limit,+ ignoreSitemap: req.body.ignoreSitemap,+ includeSubdomains: req.body.includeSubdomains,+ crawlerOptions: req.body,+ origin: req.body.origin,+ teamId: req.auth.team_id,+ plan: req.auth.plan,+ });++ // Bill the teambillTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {logger.error(`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`@@@ -234,59 -204,57 +256,11 @@@num_tokens: 0,});- return res.status(200).json({- success: true,- links: links,- scrape_id: req.body.origin?.includes("website") ? id : undefined,- });-}+ const response = {+ success: true as const,+ links: result.links,+ scrape_id: result.scrape_id+ };-// Subdomain sitemap url checking--// // For each result, check for subdomains, get their sitemaps and add them to the links-// const processedUrls = new Set();-// const processedSubdomains = new Set();--// for (const result of links) {-// let url;-// let hostParts;-// try {-// url = new URL(result);-// hostParts = url.hostname.split('.');-// } catch (e) {-// continue;-// }--// console.log("hostParts", hostParts);-// // Check if it's a subdomain (more than 2 parts, and not 'www')-// if (hostParts.length > 2 && hostParts[0] !== 'www') {-// const subdomain = hostParts[0];-// console.log("subdomain", subdomain);-// const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`;-// console.log("subdomainUrl", subdomainUrl);--// if (!processedSubdomains.has(subdomainUrl)) {-// processedSubdomains.add(subdomainUrl);--// const subdomainCrawl = crawlToCrawler(id, {-// originUrl: subdomainUrl,-// crawlerOptions: legacyCrawlerOptions(req.body),-// pageOptions: {},-// team_id: req.auth.team_id,-// createdAt: Date.now(),-// plan: req.auth.plan,-// });-// const subdomainSitemap = await subdomainCrawl.tryGetSitemap();-// if (subdomainSitemap) {-// subdomainSitemap.forEach((x) => {-// if (!processedUrls.has(x.url)) {-// processedUrls.add(x.url);-// links.push(x.url);-// }-// });-// }-// }-// }-// }+ return res.status(200).json(response);- }-- // Subdomain sitemap url checking-- // // For each result, check for subdomains, get their sitemaps and add them to the links- // const processedUrls = new Set();- // const processedSubdomains = new Set();-- // for (const result of links) {- // let url;- // let hostParts;- // try {- // url = new URL(result);- // hostParts = url.hostname.split('.');- // } catch (e) {- // continue;- // }-- // console.log("hostParts", hostParts);- // // Check if it's a subdomain (more than 2 parts, and not 'www')- // if (hostParts.length > 2 && hostParts[0] !== 'www') {- // const subdomain = hostParts[0];- // console.log("subdomain", subdomain);- // const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`;- // console.log("subdomainUrl", subdomainUrl);-- // if (!processedSubdomains.has(subdomainUrl)) {- // processedSubdomains.add(subdomainUrl);-- // const subdomainCrawl = crawlToCrawler(id, {- // originUrl: subdomainUrl,- // crawlerOptions: legacyCrawlerOptions(req.body),- // pageOptions: {},- // team_id: req.auth.team_id,- // createdAt: Date.now(),- // plan: req.auth.plan,- // });- // const subdomainSitemap = await subdomainCrawl.tryGetSitemap();- // if (subdomainSitemap) {- // subdomainSitemap.forEach((x) => {- // if (!processedUrls.has(x.url)) {- // processedUrls.add(x.url);- // links.push(x.url);- // }- // });- // }- // }- // }- // }++}diff --cc apps/api/src/controllers/v1/types.tsindex 96dea785d,2059ac8df..377c47f1d--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@@ -151,28 -151,12 +151,29 @@@ export const scrapeOptions = z.object(}).strict(strictMessage)+export type ScrapeOptions = z.infer<typeof scrapeOptions>;+export const extractV1Options = z.object({+ urls: url.array(),+ prompt: z.string().optional(),+ schema: z.any().optional(),+ limit: z.number().int().positive().finite().safe().optional(),+ ignoreSitemap: z.boolean().default(false),+ includeSubdomains: z.boolean().default(true),+ allowExternalLinks: z.boolean().default(false),+ origin: z.string().optional().default("api"),+ timeout: z.number().int().positive().finite().safe().default(60000),+}).strict(strictMessage)++export type ExtractV1Options = z.infer<typeof extractV1Options>;+export const extractRequestSchema = extractV1Options;+export type ExtractRequest = z.infer<typeof extractRequestSchema>;+- export const scrapeRequestSchema = scrapeOptions.extend({+ export const scrapeRequestSchema = scrapeOptions.omit({ timeout: true }).extend({url,origin: z.string().optional().default("api"),+ timeout: z.number().int().positive().finite().safe().default(30000),}).strict(strictMessage).refine((obj) => {const hasExtractFormat = obj.formats?.includes("extract");