Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix: skip URL rewriting for published Google Docs (#2500)
Himanshu Gupta2be7a8716506
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 0738f1f9c..0126b2e4b 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -73,6 +73,7 @@ import {import { ScrapeJobTimeoutError, CrawlDenialError } from "../../lib/error";import { htmlTransform } from "./lib/removeUnwantedElements";import { postprocessors } from "./postprocessors";+import { rewriteUrl } from "./lib/rewriteUrl";export type ScrapeUrlResponse =| {@@ -201,46 +202,6 @@ function buildFeatureFlags(return flags;}-// Convenience URL rewrites, "fake redirects" in essence.-// Used to rewrite commonly used non-scrapable URLs to their scrapable equivalents.-function rewriteUrl(url: string): string | undefined {- if (- url.startsWith("https://docs.google.com/document/d/") ||- url.startsWith("http://docs.google.com/document/d/")- ) {- const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];- if (id) {- return `https://docs.google.com/document/d/${id}/export?format=pdf`;- }- } else if (- url.startsWith("https://docs.google.com/presentation/d/") ||- url.startsWith("http://docs.google.com/presentation/d/")- ) {- const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];- if (id) {- return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;- }- } else if (- url.startsWith("https://drive.google.com/file/d/") ||- url.startsWith("http://drive.google.com/file/d/")- ) {- const id = url.match(/\/file\/d\/([-\w]+)/)?.[1];- if (id) {- return `https://drive.google.com/uc?export=download&id=${id}`;- }- } else if (- url.startsWith("https://docs.google.com/spreadsheets/d/") ||- url.startsWith("http://docs.google.com/spreadsheets/d/")- ) {- const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];- if (id) {- return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;- }- }-- return undefined;-}-// The meta object contains all required information to perform a scrape.// For example, the scrape ID, URL, options, feature flags, logs that occur while scraping.// The meta object is usually immutable, except for the logs array, and in edge cases (e.g. a new feature is suddenly required)diff --git a/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.tsnew file mode 100644index 000000000..c115c0091--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/__tests__/rewriteUrl.test.ts@@ -0,0 +1,90 @@+import { rewriteUrl } from "../rewriteUrl";++describe("rewriteUrl", () => {+ describe("Google Docs", () => {+ it("should rewrite regular Google Docs URLs to PDF export", () => {+ const url =+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/edit";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/export?format=pdf",+ );+ });++ it("should rewrite Google Docs URLs with query params", () => {+ const url =+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/edit?usp=sharing";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/export?format=pdf",+ );+ });++ it("should NOT rewrite published Google Docs URLs (/d/e/)", () => {+ const url =+ "https://docs.google.com/document/d/e/2PACX-1vTZQI1NBJsuR-LUPEcN5NyUpdfXeS9ECHx5SrtJwpBa1J0nbYkoFqP1mE-1m43ixRaGuaxnT6fnHG1h/pub";+ expect(rewriteUrl(url)).toBeUndefined();+ });++ it("should NOT rewrite published Google Docs URLs with query params", () => {+ const url =+ "https://docs.google.com/document/d/e/2PACX-1vTZQI1NBJsuR-LUPEcN5NyUpdfXeS9ECHx5SrtJwpBa1J0nbYkoFqP1mE-1m43ixRaGuaxnT6fnHG1h/pub?embedded=true";+ expect(rewriteUrl(url)).toBeUndefined();+ });++ it("should handle http:// URLs", () => {+ const url =+ "http://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/edit";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/document/d/1iqj3PY--4lSBpVkavEpjlayx0AHJDglOnJmHNOpFP1U/export?format=pdf",+ );+ });+ });++ describe("Google Presentations", () => {+ it("should rewrite regular Google Slides URLs to PDF export", () => {+ const url =+ "https://docs.google.com/presentation/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/presentation/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/export?format=pdf",+ );+ });++ it("should NOT rewrite published Google Slides URLs (/d/e/)", () => {+ const url =+ "https://docs.google.com/presentation/d/e/2PACX-1vSomePublishId/pub";+ expect(rewriteUrl(url)).toBeUndefined();+ });+ });++ describe("Google Sheets", () => {+ it("should rewrite Google Sheets URLs to HTML export", () => {+ const url =+ "https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/edit";+ expect(rewriteUrl(url)).toBe(+ "https://docs.google.com/spreadsheets/d/1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms/gviz/tq?tqx=out:html",+ );+ });++ it("should NOT rewrite published Google Sheets URLs (/d/e/)", () => {+ const url =+ "https://docs.google.com/spreadsheets/d/e/2PACX-1vSomePublishId/pubhtml";+ expect(rewriteUrl(url)).toBeUndefined();+ });+ });++ describe("Google Drive", () => {+ it("should rewrite Google Drive file URLs to download", () => {+ const url =+ "https://drive.google.com/file/d/1a2b3c4d5e6f7g8h9i0j/view?usp=sharing";+ expect(rewriteUrl(url)).toBe(+ "https://drive.google.com/uc?export=download&id=1a2b3c4d5e6f7g8h9i0j",+ );+ });+ });++ describe("Non-Google URLs", () => {+ it("should return undefined for non-Google URLs", () => {+ expect(rewriteUrl("https://example.com")).toBeUndefined();+ expect(rewriteUrl("https://firecrawl.dev")).toBeUndefined();+ });+ });+});diff --git a/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.tsnew file mode 100644index 000000000..9264c3238--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/lib/rewriteUrl.ts@@ -0,0 +1,51 @@+// Convenience URL rewrites, "fake redirects" in essence.+// Used to rewrite commonly used non-scrapable URLs to their scrapable equivalents.+export function rewriteUrl(url: string): string | undefined {+ if (+ url.startsWith("https://docs.google.com/document/d/") ||+ url.startsWith("http://docs.google.com/document/d/")+ ) {+ // Skip rewriting for published documents (/d/e/) - they're already public HTML pages+ if (url.includes("/document/d/e/")) {+ return undefined;+ }+ const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://docs.google.com/document/d/${id}/export?format=pdf`;+ }+ } else if (+ url.startsWith("https://docs.google.com/presentation/d/") ||+ url.startsWith("http://docs.google.com/presentation/d/")+ ) {+ // Skip rewriting for published presentations (/d/e/) - they're already public HTML pages+ if (url.includes("/presentation/d/e/")) {+ return undefined;+ }+ const id = url.match(/\/presentation\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;+ }+ } else if (+ url.startsWith("https://drive.google.com/file/d/") ||+ url.startsWith("http://drive.google.com/file/d/")+ ) {+ const id = url.match(/\/file\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://drive.google.com/uc?export=download&id=${id}`;+ }+ } else if (+ url.startsWith("https://docs.google.com/spreadsheets/d/") ||+ url.startsWith("http://docs.google.com/spreadsheets/d/")+ ) {+ // Skip rewriting for published spreadsheets (/d/e/) - they're already public HTML pages+ if (url.includes("/spreadsheets/d/e/")) {+ return undefined;+ }+ const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;+ }+ }++ return undefined;+}