Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: url normalization
Nicolasf2e0bfbfe304
diff --git a/apps/api/src/lib/canonical-url.ts b/apps/api/src/lib/canonical-url.tsindex fedea09d2..50570293d 100644--- a/apps/api/src/lib/canonical-url.ts+++ b/apps/api/src/lib/canonical-url.ts@@ -1,8 +1,19 @@export function normalizeUrl(url: string) {+ url = url.replace(/^https?:\/\//, "").replace(/^www\./, "");+ if (url.endsWith("/")) {+ url = url.slice(0, -1);+ }+ return url;+}++export function normalizeUrlOnlyHostname(url: string) {try {const hostname = new URL(url).hostname;return hostname.replace(/^www\./, "");} catch (error) {- return url.replace(/^https?:\/\//, "").replace(/^www\./, "").split('/')[0];+ return url+ .replace(/^https?:\/\//, "")+ .replace(/^www\./, "")+ .split("/")[0];}-}\ No newline at end of file+}diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 4fb083378..9e6f3d24e 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -51,7 +51,7 @@ import { BLOCKLISTED_URL_MESSAGE } from "../lib/strings";import { indexPage } from "../lib/extract/index/pinecone";import { Document } from "../controllers/v1/types";import { supabase_service } from "../services/supabase";-import { normalizeUrl } from "../lib/canonical-url";+import { normalizeUrl, normalizeUrlOnlyHostname } from "../lib/canonical-url";configDotenv();@@ -80,7 +80,7 @@ const gotJobInterval = Number(process.env.CONNECTION_MONITOR_INTERVAL) || 20;async function finishCrawlIfNeeded(job: Job & { id: string }, sc: StoredCrawl) {if (await finishCrawl(job.data.crawl_id)) {(async () => {- const originUrl = sc.originUrl ? normalizeUrl(sc.originUrl) : undefined;+ const originUrl = sc.originUrl ? normalizeUrlOnlyHostname(sc.originUrl) : undefined;// Get all visited URLs from Redisconst visitedUrls = await redisConnection.smembers("crawl:" + job.data.crawl_id + ":visited",