Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge remote-tracking branch 'origin/main' into nsc/new-extract

rafaelmmiller36cf49c95980
diff --cc apps/api/src/controllers/v1/map.ts
index ba7be01f2,ab9a5ed7f..8ab5c1354
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@@ -1,11 -1,6 +1,6 @@@
import { Response } from "express";
import { v4 as uuidv4 } from "uuid";
- import {
- MapDocument,
- mapRequestSchema,
- RequestWithAuth,
- scrapeOptions,
- } from "./types";
-import { mapRequestSchema, RequestWithAuth, scrapeOptions } from "./types";
++import { MapDocument, mapRequestSchema, RequestWithAuth, scrapeOptions } from "./types";
import { crawlToCrawler, StoredCrawl } from "../../lib/crawl-redis";
import { MapResponse, MapRequest } from "./types";
import { configDotenv } from "dotenv";
@@@ -30,46 -25,24 +25,48 @@@ const MAX_MAP_LIMIT = 5000
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 1000;
-export async function mapController(
- req: RequestWithAuth<{}, MapResponse, MapRequest>,
- res: Response<MapResponse>
-) {
- const startTime = new Date().getTime();
-
- req.body = mapRequestSchema.parse(req.body);
-
- const limit: number = req.body.limit ?? MAX_MAP_LIMIT;
+interface MapResult {
+ success: boolean;
+ links: string[] | any[];
+ scrape_id?: string;
+ job_id: string;
+ time_taken: number;
+}
+export async function getMapResults({
+ url,
+ search,
+ limit = MAX_MAP_LIMIT,
+ ignoreSitemap = false,
+ includeSubdomains = true,
+ crawlerOptions = {},
+ teamId,
+ plan,
+ origin,
+ includeMetadata = false,
+ allowExternalLinks
+}: {
+ url: string;
+ search?: string;
+ limit?: number;
+ ignoreSitemap?: boolean;
+ includeSubdomains?: boolean;
+ crawlerOptions?: any;
+ teamId: string;
+ plan?: string;
+ origin?: string;
+ includeMetadata?: boolean;
+ allowExternalLinks?: boolean;
+}): Promise<MapResult> {
const id = uuidv4();
- let links: string[] = [req.body.url];
+ let links: string[] = [url];
++ let mapResults: MapDocument[] = [];
const sc: StoredCrawl = {
- originUrl: req.body.url,
+ originUrl: url,
crawlerOptions: {
- ...req.body,
- limit: req.body.sitemapOnly ? 10000000 : limit,
+ ...crawlerOptions,
++ limit: crawlerOptions.sitemapOnly ? 10000000 : limit,
scrapeOptions: undefined,
},
scrapeOptions: scrapeOptions.parse({}),
@@@ -81,136 -54,130 +78,161 @@@
const crawler = crawlToCrawler(id, sc);
- let urlWithoutWww = url.replace("www.", "");
-
- let mapUrl = search && allowExternalLinks
- ? `${search} ${urlWithoutWww}`
- : search ? `${search} site:${urlWithoutWww}`
- : `site:${url}`;
-
- const resultsPerPage = 100;
- const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);
-
- const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = null;
-
- let allResults: any[] = [];
- let pagePromises: Promise<any>[] = [];
+ // If sitemapOnly is true, only get links from sitemap
- if (req.body.sitemapOnly) {
++ if (crawlerOptions.sitemapOnly) {
++ if (includeMetadata) {
++ throw new Error("includeMetadata is not supported with sitemapOnly");
++ }
+
- if (cachedResult) {
- allResults = JSON.parse(cachedResult);
- } else {
- const fetchPage = async (page: number) => {
- return fireEngineMap(mapUrl, {
- numResults: resultsPerPage,
- page: page,
+ const sitemap = await crawler.tryGetSitemap(true, true);
+ if (sitemap !== null) {
+ sitemap.forEach((x) => {
+ links.push(x.url);
});
- };
+ links = links.slice(1)
+ .map((x) => {
+ try {
+ return checkAndUpdateURLForMap(x).url.trim();
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter((x) => x !== null) as string[];
+ // links = links.slice(1, limit); // don't slice, unnecessary
+ }
+ } else {
- let urlWithoutWww = req.body.url.replace("www.", "");
++ let urlWithoutWww = url.replace("www.", "");
- pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1));
- allResults = await Promise.all(pagePromises);
- let mapUrl = req.body.search
- ? `"${req.body.search}" site:${urlWithoutWww}`
- : `site:${req.body.url}`;
++ let mapUrl = search && allowExternalLinks
++ ? `${search} ${urlWithoutWww}`
++ : search ? `${search} site:${urlWithoutWww}`
++ : `site:${url}`;
- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
- }
+ const resultsPerPage = 100;
- const maxPages = Math.ceil(
- Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage
- );
++ const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);
- console.log("allResults", allResults);
- // Parallelize sitemap fetch with serper search
- const [sitemap, ...searchResults] = await Promise.all([
- ignoreSitemap ? null : crawler.tryGetSitemap(),
- ...(cachedResult ? [] : pagePromises),
- ]);
+ const cacheKey = `fireEngineMap:${mapUrl}`;
+ const cachedResult = null;
- if (!cachedResult) {
- allResults = searchResults;
- }
+ let allResults: any[] = [];
+ let pagePromises: Promise<any>[] = [];
- if (sitemap !== null) {
- sitemap.forEach((x) => {
- links.push(x.url);
- });
- }
+ if (cachedResult) {
+ allResults = JSON.parse(cachedResult);
+ } else {
+ const fetchPage = async (page: number) => {
+ return fireEngineMap(mapUrl, {
+ numResults: resultsPerPage,
+ page: page,
+ });
+ };
+
+ pagePromises = Array.from({ length: maxPages }, (_, i) =>
+ fetchPage(i + 1)
+ );
+ allResults = await Promise.all(pagePromises);
+
+ await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
+ }
- let mapResults : MapDocument[] = allResults
- .flat()
- .filter((result) => result !== null && result !== undefined);
+ // Parallelize sitemap fetch with serper search
+ const [sitemap, ...searchResults] = await Promise.all([
- req.body.ignoreSitemap ? null : crawler.tryGetSitemap(true),
++ ignoreSitemap ? null : crawler.tryGetSitemap(),
+ ...(cachedResult ? [] : pagePromises),
+ ]);
- const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
- if (mapResults.length > minumumCutoff) {
- mapResults = mapResults.slice(0, minumumCutoff);
- }
+ if (!cachedResult) {
+ allResults = searchResults;
+ }
- if (mapResults.length > 0) {
- if (search) {
- // Ensure all map results are first, maintaining their order
- links = [
- mapResults[0].url,
- ...mapResults.slice(1).map((x) => x.url),
- ...links,
- ];
- } else {
- mapResults.map((x) => {
+ if (sitemap !== null) {
+ sitemap.forEach((x) => {
links.push(x.url);
});
}
- }
- // Perform cosine similarity between the search query and the list of links
- if (search) {
- const searchQuery = search.toLowerCase();
- links = performCosineSimilarity(links, searchQuery);
- }
- let mapResults = allResults
++ mapResults = allResults
+ .flat()
+ .filter((result) => result !== null && result !== undefined);
+
+ const minumumCutoff = Math.min(MAX_MAP_LIMIT, limit);
+ if (mapResults.length > minumumCutoff) {
+ mapResults = mapResults.slice(0, minumumCutoff);
+ }
- links = links
- .map((x) => {
- try {
- return checkAndUpdateURLForMap(x).url.trim();
- } catch (_) {
- return null;
+ if (mapResults.length > 0) {
- if (req.body.search) {
++ if (search) {
+ // Ensure all map results are first, maintaining their order
+ links = [
+ mapResults[0].url,
+ ...mapResults.slice(1).map((x) => x.url),
+ ...links,
+ ];
+ } else {
+ mapResults.map((x) => {
+ links.push(x.url);
+ });
}
- })
- .filter((x) => x !== null) as string[];
+ }
- // allows for subdomains to be included
- links = links.filter((x) => isSameDomain(x, url));
+ // Perform cosine similarity between the search query and the list of links
- if (req.body.search) {
- const searchQuery = req.body.search.toLowerCase();
-
++ if (search) {
++ const searchQuery = search.toLowerCase();
+ links = performCosineSimilarity(links, searchQuery);
+ }
- // if includeSubdomains is false, filter out subdomains
- if (!includeSubdomains) {
- links = links.filter((x) => isSameSubdomain(x, url));
- }
+ links = links
+ .map((x) => {
+ try {
+ return checkAndUpdateURLForMap(x).url.trim();
+ } catch (_) {
+ return null;
+ }
+ })
+ .filter((x) => x !== null) as string[];
+
+ // allows for subdomains to be included
- links = links.filter((x) => isSameDomain(x, req.body.url));
++ links = links.filter((x) => isSameDomain(x, url));
+
+ // if includeSubdomains is false, filter out subdomains
- if (!req.body.includeSubdomains) {
- links = links.filter((x) => isSameSubdomain(x, req.body.url));
++ if (!includeSubdomains) {
++ links = links.filter((x) => isSameSubdomain(x, url));
+ }
- // remove duplicates that could be due to http/https or www
- links = removeDuplicateUrls(links);
+ // remove duplicates that could be due to http/https or www
+ links = removeDuplicateUrls(links);
- links.slice(0, limit);
+ }
+ const linksToReturn = links.slice(0, limit);
+
+ return {
+ success: true,
+ links: includeMetadata ? mapResults : linksToReturn,
+ scrape_id: origin?.includes("website") ? id : undefined,
+ job_id: id,
+ time_taken: (new Date().getTime() - Date.now()) / 1000,
+ };
+}
+
+export async function mapController(
+ req: RequestWithAuth<{}, MapResponse, MapRequest>,
+ res: Response<MapResponse>
+) {
+ req.body = mapRequestSchema.parse(req.body);
+
+ const result = await getMapResults({
+ url: req.body.url,
+ search: req.body.search,
+ limit: req.body.limit,
+ ignoreSitemap: req.body.ignoreSitemap,
+ includeSubdomains: req.body.includeSubdomains,
+ crawlerOptions: req.body,
+ origin: req.body.origin,
+ teamId: req.auth.team_id,
+ plan: req.auth.plan,
+ });
+
+ // Bill the team
billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {
logger.error(
`Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`
@@@ -234,59 -204,57 +256,11 @@@
num_tokens: 0,
});
- return res.status(200).json({
- success: true,
- links: links,
- scrape_id: req.body.origin?.includes("website") ? id : undefined,
- });
-}
+ const response = {
+ success: true as const,
+ links: result.links,
+ scrape_id: result.scrape_id
+ };
-// Subdomain sitemap url checking
-
-// // For each result, check for subdomains, get their sitemaps and add them to the links
-// const processedUrls = new Set();
-// const processedSubdomains = new Set();
-
-// for (const result of links) {
-// let url;
-// let hostParts;
-// try {
-// url = new URL(result);
-// hostParts = url.hostname.split('.');
-// } catch (e) {
-// continue;
-// }
-
-// console.log("hostParts", hostParts);
-// // Check if it's a subdomain (more than 2 parts, and not 'www')
-// if (hostParts.length > 2 && hostParts[0] !== 'www') {
-// const subdomain = hostParts[0];
-// console.log("subdomain", subdomain);
-// const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`;
-// console.log("subdomainUrl", subdomainUrl);
-
-// if (!processedSubdomains.has(subdomainUrl)) {
-// processedSubdomains.add(subdomainUrl);
-
-// const subdomainCrawl = crawlToCrawler(id, {
-// originUrl: subdomainUrl,
-// crawlerOptions: legacyCrawlerOptions(req.body),
-// pageOptions: {},
-// team_id: req.auth.team_id,
-// createdAt: Date.now(),
-// plan: req.auth.plan,
-// });
-// const subdomainSitemap = await subdomainCrawl.tryGetSitemap();
-// if (subdomainSitemap) {
-// subdomainSitemap.forEach((x) => {
-// if (!processedUrls.has(x.url)) {
-// processedUrls.add(x.url);
-// links.push(x.url);
-// }
-// });
-// }
-// }
-// }
-// }
+ return res.status(200).json(response);
- }
-
- // Subdomain sitemap url checking
-
- // // For each result, check for subdomains, get their sitemaps and add them to the links
- // const processedUrls = new Set();
- // const processedSubdomains = new Set();
-
- // for (const result of links) {
- // let url;
- // let hostParts;
- // try {
- // url = new URL(result);
- // hostParts = url.hostname.split('.');
- // } catch (e) {
- // continue;
- // }
-
- // console.log("hostParts", hostParts);
- // // Check if it's a subdomain (more than 2 parts, and not 'www')
- // if (hostParts.length > 2 && hostParts[0] !== 'www') {
- // const subdomain = hostParts[0];
- // console.log("subdomain", subdomain);
- // const subdomainUrl = `${url.protocol}//${subdomain}.${hostParts.slice(-2).join('.')}`;
- // console.log("subdomainUrl", subdomainUrl);
-
- // if (!processedSubdomains.has(subdomainUrl)) {
- // processedSubdomains.add(subdomainUrl);
-
- // const subdomainCrawl = crawlToCrawler(id, {
- // originUrl: subdomainUrl,
- // crawlerOptions: legacyCrawlerOptions(req.body),
- // pageOptions: {},
- // team_id: req.auth.team_id,
- // createdAt: Date.now(),
- // plan: req.auth.plan,
- // });
- // const subdomainSitemap = await subdomainCrawl.tryGetSitemap();
- // if (subdomainSitemap) {
- // subdomainSitemap.forEach((x) => {
- // if (!processedUrls.has(x.url)) {
- // processedUrls.add(x.url);
- // links.push(x.url);
- // }
- // });
- // }
- // }
- // }
- // }
++}
diff --cc apps/api/src/controllers/v1/types.ts
index 96dea785d,2059ac8df..377c47f1d
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@@ -151,28 -151,12 +151,29 @@@ export const scrapeOptions = z.object(
}).strict(strictMessage)
+
export type ScrapeOptions = z.infer<typeof scrapeOptions>;
+export const extractV1Options = z.object({
+ urls: url.array(),
+ prompt: z.string().optional(),
+ schema: z.any().optional(),
+ limit: z.number().int().positive().finite().safe().optional(),
+ ignoreSitemap: z.boolean().default(false),
+ includeSubdomains: z.boolean().default(true),
+ allowExternalLinks: z.boolean().default(false),
+ origin: z.string().optional().default("api"),
+ timeout: z.number().int().positive().finite().safe().default(60000),
+}).strict(strictMessage)
+
+export type ExtractV1Options = z.infer<typeof extractV1Options>;
+export const extractRequestSchema = extractV1Options;
+export type ExtractRequest = z.infer<typeof extractRequestSchema>;
+
- export const scrapeRequestSchema = scrapeOptions.extend({
+ export const scrapeRequestSchema = scrapeOptions.omit({ timeout: true }).extend({
url,
origin: z.string().optional().default("api"),
+ timeout: z.number().int().positive().finite().safe().default(30000),
}).strict(strictMessage).refine(
(obj) => {
const hasExtractFormat = obj.formats?.includes("extract");