Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl): add parameter to treat differing query parameters as different URLs (#892)
Gergő Móricza8dc75f762fa
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex d502d1425..cb7a3ccc7 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -195,6 +195,7 @@ export async function crawlController(req: Request, res: Response) {await lockURLs(id,+ sc,jobs.map((x) => x.data.url));await addCrawlJobs(diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 9c6a288c3..b018dc999 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -76,6 +76,7 @@ export async function batchScrapeController(await lockURLs(id,+ sc,jobs.map((x) => x.data.url));await addCrawlJobs(diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex f9f60e717..aaf33f293 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -133,6 +133,7 @@ export async function crawlController(await lockURLs(id,+ sc,jobs.map((x) => x.data.url));await addCrawlJobs(diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 28dbb48f5..ec78509a1 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -205,6 +205,7 @@ const crawlerOptions = z.object({allowExternalLinks: z.boolean().default(false),ignoreSitemap: z.boolean().default(true),deduplicateSimilarURLs: z.boolean().default(true),+ ignoreQueryParameters: z.boolean().default(false),}).strict(strictMessage);// export type CrawlerOptions = {@@ -460,6 +461,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {allowExternalContentLinks: x.allowExternalLinks,ignoreSitemap: x.ignoreSitemap,deduplicateSimilarURLs: x.deduplicateSimilarURLs,+ ignoreQueryParameters: x.ignoreQueryParameters,};}@@ -474,6 +476,7 @@ export function fromLegacyCrawlerOptions(x: any): { crawlOptions: CrawlerOptionsallowExternalLinks: x.allowExternalContentLinks,ignoreSitemap: x.ignoreSitemap,deduplicateSimilarURLs: x.deduplicateSimilarURLs,+ ignoreQueryParameters: x.ignoreQueryParameters,}),internalOptions: {v0CrawlOnlyUrls: x.returnOnlyUrls,diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 34b164d2b..ddc542f92 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -90,9 +90,11 @@ export async function getThrottledJobs(teamId: string): Promise<string[]> {return await redisConnection.zrangebyscore("concurrency-limiter:" + teamId + ":throttled", Date.now(), Infinity);}-export function normalizeURL(url: string): string {+export function normalizeURL(url: string, sc: StoredCrawl): string {const urlO = new URL(url);- urlO.search = "";+ if (!sc.crawlerOptions || sc.crawlerOptions.ignoreQueryParameters) {+ urlO.search = "";+ }urlO.hash = "";return urlO.href;}@@ -130,12 +132,15 @@ export function generateURLPermutations(url: string | URL): URL[] {export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise<boolean> {if (typeof sc.crawlerOptions?.limit === "number") {- if (await redisConnection.scard("crawl:" + id + ":visited") >= sc.crawlerOptions.limit) {+ if (await redisConnection.scard("crawl:" + id + ":visited_unique") >= sc.crawlerOptions.limit) {return false;}}- url = normalizeURL(url);+ url = normalizeURL(url, sc);++ await redisConnection.sadd("crawl:" + id + ":visited_unique", url);+ await redisConnection.expire("crawl:" + id + ":visited_unique", 24 * 60 * 60, "NX");let res: boolean;if (!sc.crawlerOptions.deduplicateSimilarURLs) {@@ -150,18 +155,9 @@ export async function lockURL(id: string, sc: StoredCrawl, url: string): Promise}/// NOTE: does not check limit. only use if limit is checked beforehand e.g. with sitemap-export async function lockURLs(id: string, urls: string[]): Promise<boolean> {+export async function lockURLs(id: string, sc: StoredCrawl, urls: string[]): Promise<boolean> {urls = urls.map(url => {- try {- const urlO = new URL(url);- urlO.search = "";- urlO.hash = "";- return urlO.href;- } catch (error) {- logger.warn("Failed to normalize URL " + JSON.stringify(url) + ": " + error);- }-- return url;+ return normalizeURL(url, sc);});const res = (await redisConnection.sadd("crawl:" + id + ":visited", ...urls)) !== 0diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex e29187c2f..21dd472ea 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -320,7 +320,7 @@ async function processJob(job: Job & { id: string }, token: string) {if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (doc.metadata.url !== undefined && doc.metadata.sourceURL !== undefined && normalizeURL(doc.metadata.url) !== normalizeURL(doc.metadata.sourceURL)) {+ if (doc.metadata.url !== undefined && doc.metadata.sourceURL !== undefined && normalizeURL(doc.metadata.url, sc) !== normalizeURL(doc.metadata.sourceURL, sc)) {logger.debug("Was redirected, locking new URL...");await lockURL(job.data.crawl_id, sc, doc.metadata.url);}diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 0bd99457d..401b1c204 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -155,6 +155,7 @@ export interface CrawlParams {scrapeOptions?: CrawlScrapeOptions;webhook?: string;deduplicateSimilarURLs?: boolean;+ ignoreQueryParameters?: boolean;}/**