Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: refactor and /* glob pattern support

Nicolasd6749c211d06
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 178213d05..e6d9c7c54 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -9,15 +9,9 @@ import {
scrapeOptions,
} from "./types";
import { Document } from "../../lib/entities";
-import { StoredCrawl, crawlToCrawler } from "../../lib/crawl-redis";
-import { fireEngineMap } from "../../search/fireEngine";
import Redis from "ioredis";
import { configDotenv } from "dotenv";
import { performRanking } from "../../lib/ranker";
-import { checkAndUpdateURLForMap } from "../../lib/validateUrl";
-import { isSameDomain } from "../../lib/validateUrl";
-import { isSameSubdomain } from "../../lib/validateUrl";
-import { removeDuplicateUrls } from "../../lib/validateUrl";
import { billTeam } from "../../services/billing/credit_billing";
import { logJob } from "../../services/logging/log_job";
import { logger } from "../../lib/logger";
@@ -28,6 +22,7 @@ import { PlanType } from "../../types";
import { getJobPriority } from "../../lib/job-priority";
import { generateOpenAICompletions } from "../../scraper/scrapeURL/transformers/llmExtract";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
+import { getMapResults } from "./map";
configDotenv();
const redis = new Redis(process.env.REDIS_URL!);
@@ -43,164 +38,68 @@ export async function extractController(
req.body = extractRequestSchema.parse(req.body);
const id = crypto.randomUUID();
- let links: string[]; //= req.body.urls;
-
- const sc: StoredCrawl = {
- originUrl: req.body.urls[0],
- crawlerOptions: {
- // ...crawlerOptions,
- scrapeOptions: undefined,
- },
- scrapeOptions: scrapeOptions.parse({}),
- internalOptions: {},
- team_id: req.auth.team_id,
- createdAt: Date.now(),
- plan: req.auth.plan!,
- };
-
- const crawler = crawlToCrawler(id, sc);
-
- let urlWithoutWww = req.body.urls[0].replace("www.", "");
- console.log("urlWithoutWww", urlWithoutWww);
-
- const allowExternalLinks = req.body.allowExternalLinks ?? false;
-
- let mapUrl = req.body.prompt && allowExternalLinks
- ? `${req.body.prompt} ${urlWithoutWww}`
- : req.body.prompt ? `${req.body.prompt} site:${urlWithoutWww}`
- : `site:${urlWithoutWww}`;
-
- const resultsPerPage = 100;
- const maxPages = Math.ceil(MAX_EXTRACT_LIMIT / resultsPerPage);
-
- const cacheKey = `fireEngineMap:${mapUrl}`;
- const cachedResult = null;
-
- let allResults: any[] = [];
- let pagePromises: Promise<any>[] = [];
-
- if (cachedResult) {
- allResults = JSON.parse(cachedResult);
- } else {
- const fetchPage = async (page: number) => {
- return fireEngineMap(mapUrl, {
- numResults: resultsPerPage,
- page: page,
- });
- };
-
- pagePromises = Array.from({ length: maxPages }, (_, i) => fetchPage(i + 1));
- allResults = (await Promise.all(pagePromises)).flat();
- // console.log("allResults", allResults);
- // if allResults is empty, return an error
- if (allResults.length === 0) {
- return res.status(400).json({
- success: false,
- error: "No results found",
+ let links: string[] = [];
+ let docs: Document[] = [];
+ const earlyReturn = false;
+
+ for (const url of req.body.urls) {
+ if (url.includes('/*')) {
+ // Handle glob pattern URLs
+ const baseUrl = url.replace('/*', '');
+ const pathPrefix = baseUrl.split('/').slice(3).join('/'); // Get path after domain if any
+
+ const allowExternalLinks = req.body.allowExternalLinks ?? true;
+ let urlWithoutWww = baseUrl.replace("www.", "");
+ let mapUrl = req.body.prompt && allowExternalLinks
+ ? `${req.body.prompt} ${urlWithoutWww}`
+ : req.body.prompt ? `${req.body.prompt} site:${urlWithoutWww}`
+ : `site:${urlWithoutWww}`;
+
+ const mapResults = await getMapResults({
+ url: baseUrl,
+ search: req.body.prompt,
+ teamId: req.auth.team_id,
+ plan: req.auth.plan,
+ allowExternalLinks,
+ origin: req.body.origin,
+ limit: req.body.limit,
+ ignoreSitemap: false,
+ includeMetadata: true,
+ includeSubdomains: req.body.includeSubdomains,
});
- }
-
- await redis.set(cacheKey, JSON.stringify(allResults), "EX", 24 * 60 * 60); // Cache for 24 hours
- }
-
- // console.log("allResults", allResults);
- // Parallelize sitemap fetch with serper search
- // const [sitemap, ...searchResults] = await Promise.all([
- // req.body.ignoreSitemap ? null : null, // crawler.tryGetSitemap(),
- // ...(cachedResult ? [] : pagePromises),
- // ]);
-
- // if (!cachedResult) {
- // allResults = searchResults;
- // }
-
- links = allResults.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);
- console.log("links", links);
- // if (sitemap !== null) {
- // sitemap.forEach((x) => {
- // links.push(x.url);
- // });
- // }
- // let mapResults = allResults
- // .flat()
- // .filter((result) => result !== null && result !== undefined);
-
- // const minumumCutoff = Math.min(MAX_EXTRACT_LIMIT, req.body.limit ?? MAX_EXTRACT_LIMIT);
- // if (mapResults.length > minumumCutoff) {
- // mapResults = mapResults.slice(0, minumumCutoff);
- // }
+ let mappedLinks = mapResults.links.map(x => `url: ${x.url}, title: ${x.title}, description: ${x.description}`);
+
+ // Filter by path prefix if present
+ if (pathPrefix) {
+ mappedLinks = mappedLinks.filter(x => x.includes(`/${pathPrefix}/`));
+ }
- // if (mapResults.length > 0) {
- // if (req.body.prompt) {
- // // Ensure all map results are first, maintaining their order
- // links = [
- // mapResults[0].url,
- // ...mapResults.slice(1).map((x) => x.url),
- // ...links,
- // ];
- // } else {
- // mapResults.map((x) => {
- // links.push(x.url);
- // });
- // }
- // }
+ if (req.body.prompt) {
+ const linksAndScores = await performRanking(mappedLinks, mapUrl);
+ mappedLinks = linksAndScores
+ .filter(x => x.score > SCORE_THRESHOLD)
+ .map(x => x.link.split("url: ")[1].split(",")[0])
+ .filter(x => !isUrlBlocked(x))
+ .slice(0, MAX_RANKING_LIMIT);
+ }
- // console.log("mapResults", mapResults);
+ links.push(...mappedLinks);
- // console.log("links", links);
- let linksAndScores: { link: string; score: number }[] = [];
- // Perform cosine similarity between the search query and the list of links
- if (req.body.prompt) {
- const searchQuery = mapUrl; //req.body.prompt.toLowerCase();
- linksAndScores = await performRanking(links, searchQuery);
+ } else {
+ // Handle direct URLs without glob pattern
+ if (!isUrlBlocked(url)) {
+ links.push(url);
+ }
+ }
}
- console.log("linksAndScores", linksAndScores);
- links = linksAndScores
- .filter(x => x.score > SCORE_THRESHOLD)
- .map(x => x.link.split("url: ")[1].split(",")[0])
- .filter(x => !isUrlBlocked(x))
-
- console.log("links:", links.length);
-
- // should we use some sort of llm to determine the best links?
-
- // console.log("linksAndScores", linksAndScores);
-
- // links = links
- // .map((x) => {
- // try {
- // return checkAndUpdateURLForMap(x).url.trim();
- // } catch (_) {
- // return null;
- // }
- // })
- // .filter((x) => x !== null) as string[];
-
- // allows for subdomains to be included
- // links = links.filter((x) => isSameDomain(x, req.body.urls[0]));
-
- // if includeSubdomains is false, filter out subdomains
- // if (!req.body.includeSubdomains) {
- // links = links.filter((x) => isSameSubdomain(x, req.body.urls[0]));
- // z}
-
- // remove duplicates that could be due to http/https or www
- // links = removeDuplicateUrls(links);
-
- // get top N links
- links = links.slice(0, MAX_RANKING_LIMIT);
-
- // scrape the links
- let earlyReturn = false;
- let docs: Document[] = [];
+ // Scrape each link
for (const url of links) {
const origin = req.body.origin || "api";
const timeout = req.body.timeout ?? 30000;
const jobId = crypto.randomUUID();
- const startTime = new Date().getTime();
const jobPriority = await getJobPriority({
plan: req.auth.plan as PlanType,
team_id: req.auth.team_id,
@@ -223,11 +122,11 @@ export async function extractController(
jobPriority
);
- const totalWait = 0 //60000 // (req.body.waitFor ?? 0) + (req.body.actions ?? []).reduce((a,x) => (x.type === "wait" ? x.milliseconds ?? 0 : 0) + a, 0);
+ const totalWait = 0;
let doc: Document;
try {
- doc = await waitForJob<Document>(jobId, timeout + totalWait); // TODO: better types for this
+ doc = await waitForJob<Document>(jobId, timeout + totalWait);
} catch (e) {
logger.error(`Error in scrapeController: ${e}`);
if (e instanceof Error && (e.message.startsWith("Job wait") || e.message === "timeout")) {
@@ -245,36 +144,24 @@ export async function extractController(
await getScrapeQueue().remove(jobId);
- // const endTime = new Date().getTime();
- // const timeTakenInSeconds = (endTime - startTime) / 1000;
- // const numTokens =
- // doc && doc.extract
- // // ? numTokensFromString(doc.markdown, "gpt-3.5-turbo")
- // ? 0 // TODO: fix
- // : 0;
-
- let creditsToBeBilled = 1; // Assuming 1 credit per document
if (earlyReturn) {
- // Don't bill if we're early returning
return;
}
docs.push(doc);
}
- console.log(docs)
-
const completions = await generateOpenAICompletions(
logger.child({ method: "extractController/generateOpenAICompletions" }),
{
mode: "llm",
systemPrompt: "Only use the provided content to answer the question.",
- prompt: mapUrl,
+ prompt: req.body.prompt,
schema: req.body.schema,
},
docs.map(x => x.markdown).join('\n')
);
- console.log("completions", completions);
+ // console.log("completions", completions);
// if(req.body.extract && req.body.formats.includes("extract")) {
// creditsToBeBilled = 5;
@@ -355,7 +242,7 @@ export async function extractController(
return res.status(200).json({
success: true,
- data: data, // includeMetadata ? mapResults : linksToReturn,
- scrape_id: id, //origin?.includes("website") ? id : undefined,
+ data: data,
+ scrape_id: id,
});
}
\ No newline at end of file
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 91d712de2..f2e9453a6 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -29,6 +29,14 @@ const MAX_MAP_LIMIT = 5000;
// Max Links that "Smart /map" can return
const MAX_FIRE_ENGINE_RESULTS = 1000;
+interface MapResult {
+ success: boolean;
+ links: string[] | any[];
+ scrape_id?: string;
+ job_id: string;
+ time_taken: number;
+}
+
export async function getMapResults({
url,
search,
@@ -39,8 +47,8 @@ export async function getMapResults({
teamId,
plan,
origin,
- subId,
- includeMetadata = false
+ includeMetadata = false,
+ allowExternalLinks
}: {
url: string;
search?: string;
@@ -51,9 +59,9 @@ export async function getMapResults({
teamId: string;
plan?: string;
origin?: string;
- subId: string | null;
includeMetadata?: boolean;
-}) {
+ allowExternalLinks?: boolean;
+}): Promise<MapResult> {
const id = uuidv4();
let links: string[] = [url];
@@ -74,10 +82,11 @@ export async function getMapResults({
let urlWithoutWww = url.replace("www.", "");
- let mapUrl = search
- ? `"${search}" site:${urlWithoutWww}`
+ let mapUrl = search && allowExternalLinks
+ ? `${search} ${urlWithoutWww}`
+ : search ? `${search} site:${urlWithoutWww}`
: `site:${url}`;
-
+
const resultsPerPage = 100;
const maxPages = Math.ceil(Math.min(MAX_FIRE_ENGINE_RESULTS, limit) / resultsPerPage);
@@ -171,34 +180,14 @@ export async function getMapResults({
// remove duplicates that could be due to http/https or www
links = removeDuplicateUrls(links);
- billTeam(teamId, subId, 1).catch((error) => {
- logger.error(
- `Failed to bill team ${teamId} for 1 credit: ${error}`
- );
- });
-
const linksToReturn = links.slice(0, limit);
- logJob({
- job_id: id,
- success: links.length > 0,
- message: "Map completed",
- num_docs: linksToReturn.length,
- docs: linksToReturn,
- time_taken: (new Date().getTime() - Date.now()) / 1000,
- team_id: teamId,
- mode: "map",
- url: url,
- crawlerOptions: {},
- scrapeOptions: {},
- origin: origin ?? "api",
- num_tokens: 0,
- });
-
return {
success: true,
links: includeMetadata ? mapResults : linksToReturn,
scrape_id: origin?.includes("website") ? id : undefined,
+ job_id: id,
+ time_taken: (new Date().getTime() - Date.now()) / 1000,
};
}
@@ -208,7 +197,6 @@ export async function mapController(
) {
req.body = mapRequestSchema.parse(req.body);
- console.log("req.body", req.body);
const result = await getMapResults({
url: req.body.url,
search: req.body.search,
@@ -216,10 +204,33 @@ export async function mapController(
ignoreSitemap: req.body.ignoreSitemap,
includeSubdomains: req.body.includeSubdomains,
crawlerOptions: req.body,
+ origin: req.body.origin,
teamId: req.auth.team_id,
plan: req.auth.plan,
- origin: req.body.origin,
- subId: req.acuc?.sub_id
+ });
+
+ // Bill the team
+ billTeam(req.auth.team_id, req.acuc?.sub_id, 1).catch((error) => {
+ logger.error(
+ `Failed to bill team ${req.auth.team_id} for 1 credit: ${error}`
+ );
+ });
+
+ // Log the job
+ logJob({
+ job_id: result.job_id,
+ success: result.links.length > 0,
+ message: "Map completed",
+ num_docs: result.links.length,
+ docs: result.links,
+ time_taken: result.time_taken,
+ team_id: req.auth.team_id,
+ mode: "map",
+ url: req.body.url,
+ crawlerOptions: {},
+ scrapeOptions: {},
+ origin: req.body.origin ?? "api",
+ num_tokens: 0,
});
const response = {