Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL): separate URL rewrites to different function

Gergő Móricz7ed19c0ac0e8
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 65b697b42..ba8e3256c 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -128,6 +128,24 @@ function buildFeatureFlags(
return flags;
}
+// Convenience URL rewrites, "fake redirects" in essence.
+// Used to rewrite commonly used non-scrapable URLs to their scrapable equivalents.
+function rewriteUrl(url: string): string | undefined {
+ if (url.startsWith("https://docs.google.com/document/d/") || url.startsWith("http://docs.google.com/document/d/")) {
+ const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://docs.google.com/document/d/${id}/export?format=pdf`;
+ }
+ } else if (url.startsWith("https://docs.google.com/presentation/d/") || url.startsWith("http://docs.google.com/presentation/d/")) {
+ const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;
+ }
+ }
+
+ return undefined;
+}
+
// The meta object contains all required information to perform a scrape.
// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.
// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)
@@ -157,23 +175,10 @@ async function buildMetaObject(
});
const logs: any[] = [];
- let rewrittenUrl: string | undefined;
- if (url.startsWith("https://docs.google.com/document/d/") || url.startsWith("http://docs.google.com/document/d/")) {
- const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];
- if (id) {
- rewrittenUrl = `https://docs.google.com/document/d/${id}/export?format=pdf`;
- }
- } else if (url.startsWith("https://docs.google.com/presentation/d/") || url.startsWith("http://docs.google.com/presentation/d/")) {
- const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];
- if (id) {
- rewrittenUrl = `https://docs.google.com/presentation/d/${id}/export?format=pdf`;
- }
- }
-
return {
id,
url,
- rewrittenUrl,
+ rewrittenUrl: rewriteUrl(url),
options,
internalOptions,
logger,