Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrapeURL/rewrite): scrape Google Drive TXT/PDF files and sheets (#2053)

Gergő Móricz822b95f0ba86
diff --git a/apps/api/src/__tests__/snips/v1/scrape.test.ts b/apps/api/src/__tests__/snips/v1/scrape.test.ts
index 44236a10a..c49581e1f 100644
--- a/apps/api/src/__tests__/snips/v1/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v1/scrape.test.ts
@@ -937,11 +937,14 @@ describe("Scrape tests", () => {
// text on the last page
expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");
}, scrapeTimeout * 5);
+ });
+ }
+ describe("URL rewriting", () => {
+ if (!process.env.TEST_SUITE_SELF_HOSTED) {
it.concurrent("scrapes Google Docs links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",
- timeout: scrapeTimeout * 5,
}, identity);
expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");
@@ -950,13 +953,36 @@ describe("Scrape tests", () => {
it.concurrent("scrapes Google Slides links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/presentation/d/1pDKL1UULpr6siq_eVWE1hjqt5MKCgSSuKS_MWahnHAQ/view",
- timeout: scrapeTimeout * 5,
}, identity);
expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");
}, scrapeTimeout * 5);
- });
- }
+
+ it.concurrent("scrapes Google Drive PDF files as PDFs", async () => {
+ const response = await scrape({
+ url: "https://drive.google.com/file/d/1QrgvXM2F7sgSdrhoBfdp9IMBVhUk-Ueu/view?usp=drive_link",
+ }, identity);
+
+ expect(response.markdown).toContain("This is a simple PDF file.");
+ }, scrapeTimeout * 5);
+ }
+
+ it.concurrent("scrapes Google Drive text files correctly", async () => {
+ const response = await scrape({
+ url: "https://drive.google.com/file/d/14m3ZVDnWJwwPSDHX6U6jkL7FXxOf6cHB/view?usp=sharing",
+ }, identity);
+
+ expect(response.markdown).toContain("This is a simple TXT file.");
+ }, scrapeTimeout * 5);
+
+ it.concurrent("scrapes Google Sheets links correctly", async () => {
+ const response = await scrape({
+ url: "https://docs.google.com/spreadsheets/d/1DTpw_bbsf3OY17ZqEYEpW6lAmLdCRC2WfLrV0isG9ac/edit?usp=sharing",
+ }, identity);
+
+ expect(response.markdown).toContain("This is a test sheet.");
+ }, scrapeTimeout * 5);
+ });
if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {
describe("JSON format", () => {
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index 315f93cca..9de6f38e6 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -851,10 +851,15 @@ describe("Scrape tests", () => {
// text on the last page
expect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");
}, scrapeTimeout * 5);
+ });
+ }
+ describe("URL rewriting", () => {
+ if (!process.env.TEST_SUITE_SELF_HOSTED) {
it.concurrent("scrapes Google Docs links as PDFs", async () => {
const response = await scrape({
url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",
+ maxAge: 0,
}, identity);
expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");
@@ -868,8 +873,35 @@ describe("Scrape tests", () => {
expect(response.markdown).toContain("This is a test to confirm Google Slides scraping abilities.");
}, scrapeTimeout * 5);
- });
- }
+
+ it.concurrent("scrapes Google Drive PDF files as PDFs", async () => {
+ const response = await scrape({
+ url: "https://drive.google.com/file/d/1QrgvXM2F7sgSdrhoBfdp9IMBVhUk-Ueu/view?usp=drive_link",
+ maxAge: 0,
+ }, identity);
+
+ expect(response.markdown).toContain("This is a simple PDF file.");
+ }, scrapeTimeout * 5);
+ }
+
+ it.concurrent("scrapes Google Drive text files correctly", async () => {
+ const response = await scrape({
+ url: "https://drive.google.com/file/d/14m3ZVDnWJwwPSDHX6U6jkL7FXxOf6cHB/view?usp=sharing",
+ maxAge: 0,
+ }, identity);
+
+ expect(response.markdown).toContain("This is a simple TXT file.");
+ }, scrapeTimeout * 5);
+
+ it.concurrent("scrapes Google Sheets links correctly", async () => {
+ const response = await scrape({
+ url: "https://docs.google.com/spreadsheets/d/1DTpw_bbsf3OY17ZqEYEpW6lAmLdCRC2WfLrV0isG9ac/edit?usp=sharing",
+ maxAge: 0,
+ }, identity);
+
+ expect(response.markdown).toContain("This is a test sheet.");
+ }, scrapeTimeout * 5);
+ });
if (!process.env.TEST_SUITE_SELF_HOSTED || process.env.OPENAI_API_KEY || process.env.OLLAMA_BASE_URL) {
describe("JSON format", () => {
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index c67581cd4..6b782f999 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -38,7 +38,8 @@ export async function specialtyScrapeCheck(
});
} else if (
contentType === "application/pdf" ||
- contentType.startsWith("application/pdf;")
+ contentType.startsWith("application/pdf;") ||
+ (contentType === "application/octet-stream" && (feRes?.file?.content.startsWith("JVBERi0") || feRes?.content.startsWith("%PDF-")))
) {
// .pdf
throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index b98448a86..6f811a049 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -154,6 +154,16 @@ function rewriteUrl(url: string): string | undefined {
if (id) {
return `https://docs.google.com/presentation/d/${id}/export?format=pdf`;
}
+ } else if (url.startsWith("https://drive.google.com/file/d/") || url.startsWith("http://drive.google.com/file/d/")) {
+ const id = url.match(/\/file\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://drive.google.com/uc?export=download&id=${id}`;
+ }
+ } else if (url.startsWith("https://docs.google.com/spreadsheets/d/") || url.startsWith("http://docs.google.com/spreadsheets/d/")) {
+ const id = url.match(/\/spreadsheets\/d\/([-\w]+)/)?.[1];
+ if (id) {
+ return `https://docs.google.com/spreadsheets/d/${id}/gviz/tq?tqx=out:html`;
+ }
}
return undefined;