Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-redis/generateURLPermutations): dedupe index.html/index.php/slash/bare URL ends (FIR-827) (#1134)
Gergő Móricz34fe360dc1ac
diff --git a/apps/api/src/lib/crawl-redis.test.ts b/apps/api/src/lib/crawl-redis.test.tsindex 65d4e13ab..f27598b10 100644--- a/apps/api/src/lib/crawl-redis.test.ts+++ b/apps/api/src/lib/crawl-redis.test.ts@@ -5,37 +5,69 @@ describe("generateURLPermutations", () => {const bareHttps = generateURLPermutations("https://firecrawl.dev").map((x) => x.href,);- expect(bareHttps.length).toBe(4);+ expect(bareHttps.length).toBe(16);expect(bareHttps.includes("https://firecrawl.dev/")).toBe(true);+ expect(bareHttps.includes("https://firecrawl.dev/index.html")).toBe(true);+ expect(bareHttps.includes("https://firecrawl.dev/index.php")).toBe(true);expect(bareHttps.includes("https://www.firecrawl.dev/")).toBe(true);+ expect(bareHttps.includes("https://www.firecrawl.dev/index.html")).toBe(true);+ expect(bareHttps.includes("https://www.firecrawl.dev/index.php")).toBe(true);expect(bareHttps.includes("http://firecrawl.dev/")).toBe(true);+ expect(bareHttps.includes("http://firecrawl.dev/index.html")).toBe(true);+ expect(bareHttps.includes("http://firecrawl.dev/index.php")).toBe(true);expect(bareHttps.includes("http://www.firecrawl.dev/")).toBe(true);+ expect(bareHttps.includes("http://www.firecrawl.dev/index.html")).toBe(true);+ expect(bareHttps.includes("http://www.firecrawl.dev/index.php")).toBe(true);const bareHttp = generateURLPermutations("http://firecrawl.dev").map((x) => x.href,);- expect(bareHttp.length).toBe(4);+ expect(bareHttp.length).toBe(16);expect(bareHttp.includes("https://firecrawl.dev/")).toBe(true);+ expect(bareHttp.includes("https://firecrawl.dev/index.html")).toBe(true);+ expect(bareHttp.includes("https://firecrawl.dev/index.php")).toBe(true);expect(bareHttp.includes("https://www.firecrawl.dev/")).toBe(true);+ expect(bareHttp.includes("https://www.firecrawl.dev/index.html")).toBe(true);+ expect(bareHttp.includes("https://www.firecrawl.dev/index.php")).toBe(true);expect(bareHttp.includes("http://firecrawl.dev/")).toBe(true);+ expect(bareHttp.includes("http://firecrawl.dev/index.html")).toBe(true);+ expect(bareHttp.includes("http://firecrawl.dev/index.php")).toBe(true);expect(bareHttp.includes("http://www.firecrawl.dev/")).toBe(true);+ expect(bareHttp.includes("http://www.firecrawl.dev/index.html")).toBe(true);+ expect(bareHttp.includes("http://www.firecrawl.dev/index.php")).toBe(true);const wwwHttps = generateURLPermutations("https://www.firecrawl.dev").map((x) => x.href,);- expect(wwwHttps.length).toBe(4);+ expect(wwwHttps.length).toBe(16);expect(wwwHttps.includes("https://firecrawl.dev/")).toBe(true);+ expect(wwwHttps.includes("https://firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttps.includes("https://firecrawl.dev/index.php")).toBe(true);expect(wwwHttps.includes("https://www.firecrawl.dev/")).toBe(true);+ expect(wwwHttps.includes("https://www.firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttps.includes("https://www.firecrawl.dev/index.php")).toBe(true);expect(wwwHttps.includes("http://firecrawl.dev/")).toBe(true);+ expect(wwwHttps.includes("http://firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttps.includes("http://firecrawl.dev/index.php")).toBe(true);expect(wwwHttps.includes("http://www.firecrawl.dev/")).toBe(true);+ expect(wwwHttps.includes("http://www.firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttps.includes("http://www.firecrawl.dev/index.php")).toBe(true);const wwwHttp = generateURLPermutations("http://www.firecrawl.dev").map((x) => x.href,);- expect(wwwHttp.length).toBe(4);+ expect(wwwHttp.length).toBe(16);expect(wwwHttp.includes("https://firecrawl.dev/")).toBe(true);+ expect(wwwHttp.includes("https://firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttp.includes("https://firecrawl.dev/index.php")).toBe(true);expect(wwwHttp.includes("https://www.firecrawl.dev/")).toBe(true);+ expect(wwwHttp.includes("https://www.firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttp.includes("https://www.firecrawl.dev/index.php")).toBe(true);expect(wwwHttp.includes("http://firecrawl.dev/")).toBe(true);+ expect(wwwHttp.includes("http://firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttp.includes("http://firecrawl.dev/index.php")).toBe(true);expect(wwwHttp.includes("http://www.firecrawl.dev/")).toBe(true);+ expect(wwwHttp.includes("http://www.firecrawl.dev/index.html")).toBe(true);+ expect(wwwHttp.includes("http://www.firecrawl.dev/index.php")).toBe(true);});});diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 553934d7a..c08705868 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -229,6 +229,34 @@ export function generateURLPermutations(url: string | URL): URL[] {return [urlWithHTTP, urlWithHTTPS];});+ // Construct more versions for index.html/index.php+ permutations = permutations.flatMap((urlO) => {+ const urlWithHTML = new URL(urlO);+ const urlWithPHP = new URL(urlO);+ const urlWithBare = new URL(urlO);+ const urlWithSlash = new URL(urlO);++ if (urlO.pathname.endsWith("/")) {+ urlWithBare.pathname = urlWithBare.pathname.length === 1 ? urlWithBare.pathname : urlWithBare.pathname.slice(0, -1);+ urlWithHTML.pathname += "index.html";+ urlWithPHP.pathname += "index.php";+ } else if (urlO.pathname.endsWith("/index.html")) {+ urlWithPHP.pathname = urlWithPHP.pathname.slice(0, -"index.html".length) + "index.php";+ urlWithSlash.pathname = urlWithSlash.pathname.slice(0, -"index.html".length);+ urlWithBare.pathname = urlWithBare.pathname.slice(0, -"/index.html".length);+ } else if (urlO.pathname.endsWith("/index.php")) {+ urlWithHTML.pathname = urlWithHTML.pathname.slice(0, -"index.php".length) + "index.html";+ urlWithSlash.pathname = urlWithSlash.pathname.slice(0, -"index.php".length);+ urlWithBare.pathname = urlWithBare.pathname.slice(0, -"/index.php".length);+ } else {+ urlWithSlash.pathname += "/";+ urlWithHTML.pathname += "/index.html";+ urlWithPHP.pathname += "/index.php";+ }++ return [urlWithHTML, urlWithPHP, urlWithSlash, urlWithBare];+ });+return permutations;}