Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): separate URL rewrites to different function
Gergő Móricz7ed19c0ac0e8
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 65b697b42..ba8e3256c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -128,6 +128,24 @@ function buildFeatureFlags(return flags;}+// Convenience URL rewrites, "fake redirects" in essence.+// Used to rewrite commonly used non-scrapable URLs to their scrapable equivalents.+function rewriteUrl(url: string): string | undefined {+ if (url.startsWith("https://docs.google.com/document/d/") || url.startsWith("http://docs.google.com/document/d/")) {+ const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://docs.google.com/document/d/${id}/export?format=pdf`;+ }+ } else if (url.startsWith("https://docs.google.com/presentation/d/") || url.startsWith("http://docs.google.com/presentation/d/")) {+ const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;+ }+ }++ return undefined;+}+// The meta object contains all required information to perform a scrape.// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)@@ -157,23 +175,10 @@ async function buildMetaObject(});const logs: any[] = [];- let rewrittenUrl: string | undefined;- if (url.startsWith("https://docs.google.com/document/d/") || url.startsWith("http://docs.google.com/document/d/")) {- const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];- if (id) {- rewrittenUrl = `https://docs.google.com/document/d/${id}/export?format=pdf`;- }- } else if (url.startsWith("https://docs.google.com/presentation/d/") || url.startsWith("http://docs.google.com/presentation/d/")) {- const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];- if (id) {- rewrittenUrl = `https://docs.google.com/presentation/d/${id}/export?format=pdf`;- }- }-return {id,url,- rewrittenUrl,+ rewrittenUrl: rewriteUrl(url),options,internalOptions,logger,