Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix: skip URL rewriting for published Google Docs (#2500)

Himanshu Gupta2be7a8716506
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 0738f1f9c..0126b2e4b 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -73,6 +73,7 @@ import {
import { ScrapeJobTimeoutError, CrawlDenialError } from "../../lib/error";
import { htmlTransform } from "./lib/removeUnwantedElements";
import { postprocessors } from "./postprocessors";
+import { rewriteUrl } from "./lib/rewriteUrl";
export type ScrapeUrlResponse =
| {
@@ -201,46 +202,6 @@ function buildFeatureFlags(
return flags;
}
-// Convenience URL rewrites, "fake redirects" in essence.
-// Used to rewrite commonly used non-scrapable URLs to their scrapable equivalents.
-function rewriteUrl(url: string): string | undefined {
- if (
- url.startsWith("https://docs.google.com/document/d/") ||
- url.startsWith("http://docs.google.com/document/d/")
- ) {
- const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];
- if (id) {
- return `https://docs.google.com/document/d/${id}/export?format=pdf`;
- }
- } else if (
- url.startsWith("https://docs.google.com/presentation/d/") ||
- url.startsWith("http://docs.google.com/presentation/d/")
- ) {
- const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];
- if (id) {
- return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;
- }
- } else if (
- url.startsWith("https://drive.google.com/file/d/") ||
- url.startsWith("http://drive.google.com/file/d/")
- ) {
- const id = url.match(/\/file\/d\/([-\w]+)/)?.[1];
- if (id) {
- return `https://drive.google.com/uc?export=download&id=${id}`;
- }
- } else if (
- url.startsWith("https://docs.google.com/spreadsheets/d/") ||
- url.startsWith("http://docs.google.com/spreadsheets/d/")
- ) {
- const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];
- if (id) {
- return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;
- }
- }
-
- return undefined;
-}
-
// The meta object contains all required information to perform a scrape.
// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.
// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)
diff --git a/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts
new file mode 100644
index 000000000..c115c0091
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts
@@ -0,0 +1,90 @@
+import { rewriteUrl } from "../rewriteUrl";
+
+describe("rewriteUrl", () => {
+ describe("Google Docs", () => {
+ it("should rewrite regular Google Docs URLs to PDF export", () => {
+ const url =
+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/edit";
+ expect(rewriteUrl(url)).toBe(
+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/export?format=pdf",
+ );
+ });
+
+ it("should rewrite Google Docs URLs with query params", () => {
+ const url =
+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/edit?usp=sharing";
+ expect(rewriteUrl(url)).toBe(
+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/export?format=pdf",
+ );
+ });
+
+ it("should NOT rewrite published Google Docs URLs (/d/e/)", () => {
+ const url =
+ "https://docs.google.com/document/d/e/2PACX-1vTZQI1NBJsuR-LUPEcN5NyUpdfXeS9ECHx5SrtJwpBa1J0nbYkoFqP1mE-1m43ixRaGuaxnT6fnHG1h/pub";
+ expect(rewriteUrl(url)).toBeUndefined();
+ });
+
+ it("should NOT rewrite published Google Docs URLs with query params", () => {
+ const url =
+ "https://docs.google.com/document/d/e/2PACX-1vTZQI1NBJsuR-LUPEcN5NyUpdfXeS9ECHx5SrtJwpBa1J0nbYkoFqP1mE-1m43ixRaGuaxnT6fnHG1h/pub?embedded=true";
+ expect(rewriteUrl(url)).toBeUndefined();
+ });
+
+ it("should handle http:// URLs", () => {
+ const url =
+ "http://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/edit";
+ expect(rewriteUrl(url)).toBe(
+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/export?format=pdf",
+ );
+ });
+ });
+
+ describe("Google Presentations", () => {
+ it("should rewrite regular Google Slides URLs to PDF export", () => {
+ const url =
+ "https://docs.google.com/presentation/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit";
+ expect(rewriteUrl(url)).toBe(
+ "https://docs.google.com/presentation/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/export?format=pdf",
+ );
+ });
+
+ it("should NOT rewrite published Google Slides URLs (/d/e/)", () => {
+ const url =
+ "https://docs.google.com/presentation/d/e/2PACX-1vSomePublishId/pub";
+ expect(rewriteUrl(url)).toBeUndefined();
+ });
+ });
+
+ describe("Google Sheets", () => {
+ it("should rewrite Google Sheets URLs to HTML export", () => {
+ const url =
+ "https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit";
+ expect(rewriteUrl(url)).toBe(
+ "https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/gviz/tq?tqx=out:html",
+ );
+ });
+
+ it("should NOT rewrite published Google Sheets URLs (/d/e/)", () => {
+ const url =
+ "https://docs.google.com/spreadsheets/d/e/2PACX-1vSomePublishId/pubhtml";
+ expect(rewriteUrl(url)).toBeUndefined();
+ });
+ });
+
+ describe("Google Drive", () => {
+ it("should rewrite Google Drive file URLs to download", () => {
+ const url =
+ "https://drive.google.com/file/d/1a2b3c4d5e6f7g8h9i0j/view?usp=sharing";
+ expect(rewriteUrl(url)).toBe(
+ "https://drive.google.com/uc?export=download&id=1a2b3c4d5e6f7g8h9i0j",
+ );
+ });
+ });
+
+ describe("Non-Google URLs", () => {
+ it("should return undefined for non-Google URLs", () => {
+ expect(rewriteUrl("https://example.com")).toBeUndefined();
+ expect(rewriteUrl("https://firecrawl.dev")).toBeUndefined();
+ });
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts
new file mode 100644
index 000000000..9264c3238
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts
@@ -0,0 +1,51 @@
+// Convenience URL rewrites, "fake redirects" in essence.
+// Used to rewrite commonly used non-scrapable URLs to their scrapable equivalents.
+export function rewriteUrl(url: string): string | undefined {
+ if (
+ url.startsWith("https://docs.google.com/document/d/") ||
+ url.startsWith("http://docs.google.com/document/d/")
+ ) {
+ // Skip rewriting for published documents (/d/e/) - they're already public HTML pages
+ if (url.includes("/document/d/e/")) {
+ return undefined;
+ }
+ const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://docs.google.com/document/d/${id}/export?format=pdf`;
+ }
+ } else if (
+ url.startsWith("https://docs.google.com/presentation/d/") ||
+ url.startsWith("http://docs.google.com/presentation/d/")
+ ) {
+ // Skip rewriting for published presentations (/d/e/) - they're already public HTML pages
+ if (url.includes("/presentation/d/e/")) {
+ return undefined;
+ }
+ const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;
+ }
+ } else if (
+ url.startsWith("https://drive.google.com/file/d/") ||
+ url.startsWith("http://drive.google.com/file/d/")
+ ) {
+ const id = url.match(/\/file\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://drive.google.com/uc?export=download&id=${id}`;
+ }
+ } else if (
+ url.startsWith("https://docs.google.com/spreadsheets/d/") ||
+ url.startsWith("http://docs.google.com/spreadsheets/d/")
+ ) {
+ // Skip rewriting for published spreadsheets (/d/e/) - they're already public HTML pages
+ if (url.includes("/spreadsheets/d/e/")) {
+ return undefined;
+ }
+ const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;
+ }
+ }
+
+ return undefined;
+}