Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL/rewrite): scrape Google Drive TXT/PDF files and sheets (#2053)
Gergő Móricz822b95f0ba86
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.tsindex 44236a10a..c49581e1f 100644--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts@@ -937,11 +937,14 @@ describe("Scrape tests", () => {// text on the last pageexpect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");}, scrapeTimeout * 5);+ });+ }+ describe("URL rewriting", () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED) {it.concurrent("scrapes Google Docs links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",- timeout: scrapeTimeout * 5,}, identity);expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");@@ -950,13 +953,36 @@ describe("Scrape tests", () => {it.concurrent("scrapes Google Slides links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",- timeout: scrapeTimeout * 5,}, identity);expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");}, scrapeTimeout * 5);- });- }++ it.concurrent("scrapes Google Drive PDF files as PDFs", async () => {+ const response = await scrape({+ url: "https://drive.google.com/file/d/1QrgvXM2F7sgSdrhoBfdp9IMBVhUk-Ueu/view?usp=drive_link",+ }, identity);++ expect(response.markdown).toContain("This is a simple PDF file.");+ }, scrapeTimeout * 5);+ }++ it.concurrent("scrapes Google Drive text files correctly", async () => {+ const response = await scrape({+ url: "https://drive.google.com/file/d/14m3ZVDnWJwwPSDHX6U6jkL7FXxOf6cHB/view?usp=sharing",+ }, identity);++ expect(response.markdown).toContain("This is a simple TXT file.");+ }, scrapeTimeout * 5);++ it.concurrent("scrapes Google Sheets links correctly", async () => {+ const response = await scrape({+ url: "https://docs.google.com/spreadsheets/d/1DTpw_bbsf3OY17ZqEYEpW6lAmLdCRC2WfLrV0isG9ac/edit?usp=sharing",+ }, identity);++ expect(response.markdown).toContain("This is a test sheet.");+ }, scrapeTimeout * 5);+ });if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {describe("JSON format", () => {diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.tsindex 315f93cca..9de6f38e6 100644--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts@@ -851,10 +851,15 @@ describe("Scrape tests", () => {// text on the last pageexpect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");}, scrapeTimeout * 5);+ });+ }+ describe("URL rewriting", () => {+ if (!process.env.TEST_SUITE_SELF_HOSTED) {it.concurrent("scrapes Google Docs links as PDFs", async () => {const response = await scrape({url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",+ maxAge: 0,}, identity);expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");@@ -868,8 +873,35 @@ describe("Scrape tests", () => {expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");}, scrapeTimeout * 5);- });- }++ it.concurrent("scrapes Google Drive PDF files as PDFs", async () => {+ const response = await scrape({+ url: "https://drive.google.com/file/d/1QrgvXM2F7sgSdrhoBfdp9IMBVhUk-Ueu/view?usp=drive_link",+ maxAge: 0,+ }, identity);++ expect(response.markdown).toContain("This is a simple PDF file.");+ }, scrapeTimeout * 5);+ }++ it.concurrent("scrapes Google Drive text files correctly", async () => {+ const response = await scrape({+ url: "https://drive.google.com/file/d/14m3ZVDnWJwwPSDHX6U6jkL7FXxOf6cHB/view?usp=sharing",+ maxAge: 0,+ }, identity);++ expect(response.markdown).toContain("This is a simple TXT file.");+ }, scrapeTimeout * 5);++ it.concurrent("scrapes Google Sheets links correctly", async () => {+ const response = await scrape({+ url: "https://docs.google.com/spreadsheets/d/1DTpw_bbsf3OY17ZqEYEpW6lAmLdCRC2WfLrV0isG9ac/edit?usp=sharing",+ maxAge: 0,+ }, identity);++ expect(response.markdown).toContain("This is a test sheet.");+ }, scrapeTimeout * 5);+ });if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {describe("JSON format", () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex c67581cd4..6b782f999 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -38,7 +38,8 @@ export async function specialtyScrapeCheck(});} else if (contentType === "application/pdf" ||- contentType.startsWith("application/pdf;")+ contentType.startsWith("application/pdf;") ||+ (contentType === "application/octet-stream" && (feRes?.file?.content.startsWith("JVBERi0") || feRes?.content.startsWith("%PDF-")))) {throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex b98448a86..6f811a049 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -154,6 +154,16 @@ function rewriteUrl(url: string): string | undefined {if (id) {return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;}+ } else if (url.startsWith("https://drive.google.com/file/d/") || url.startsWith("http://drive.google.com/file/d/")) {+ const id = url.match(/\/file\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://drive.google.com/uc?export=download&id=${id}`;+ }+ } else if (url.startsWith("https://docs.google.com/spreadsheets/d/") || url.startsWith("http://docs.google.com/spreadsheets/d/")) {+ const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];+ if (id) {+ return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;+ }}return undefined;