Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scraper): improve charset detection regex to accurately parse meta tags (#1265)
Grass Huang7bf04d409aa8
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 642690487..687c8232e 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -74,7 +74,7 @@ export async function scrapeURLWithFetch(const buf = Buffer.from(await x.arrayBuffer());let text = buf.toString("utf8");- const charset = (text.match(/charset=["']?(.+?)["']?>/) ?? [])[1]+ const charset = (text.match(/<meta\b[^>]*charset\s*=\s*["']?([^"'\s\/>]+)/i) ?? [])[1]try {if (charset) {text = new TextDecoder(charset.trim()).decode(buf);