Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): vue + angular router url normalization
delong36d537b8e9448
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 6bc0875f2..3f102d7e0 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -308,7 +308,11 @@ export function normalizeURL(url: string, sc: StoredCrawl): string {urlO.search = "";}// allow hash-based routes- if (!urlO.hash || urlO.hash.length <= 2 || !urlO.hash.startsWith("#/")) {+ if (+ !urlO.hash ||+ urlO.hash.length <= 2 ||+ (!urlO.hash.startsWith("#/") && !urlO.hash.startsWith("#!/"))+ ) {urlO.hash = "";}return urlO.href;diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex e56fcd7d5..c262dfc99 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -209,7 +209,15 @@ export const useSearchIndex =export function normalizeURLForIndex(url: string): string {const urlObj = new URL(url);- urlObj.hash = "";++ if (+ !urlObj.hash ||+ urlObj.hash.length <= 2 ||+ (!urlObj.hash.startsWith("#/") && !urlObj.hash.startsWith("#!/"))+ ) {+ urlObj.hash = "";+ }+urlObj.protocol = "https";if (urlObj.port === "80" || urlObj.port === "443") {