Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrape): support Google Docs (FIR-1365) (#1686)
Gergő Móriczf93942826439
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 6361f988a..1336a87bd 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -745,6 +745,15 @@ describe("Scrape tests", () => {// text on the last pageexpect(response.markdown).toContain("Redistribution and use in source and binary forms, with or without modification");}, 310000);++ it.concurrent("scrapes Google Docs links as PDFs", async () => {+ const response = await scrape({+ url: "https://docs.google.com/document/d/1H-hOLYssS8xXl2o5hxj4ipE7yyhZAX1s7ADYM1Hdlzo/view",+ timeout: 300000,+ });++ expect(response.markdown).toContain("This is a test to confirm Google Docs scraping abilities.");+ }, 310000);});}diff --git a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts b/apps/api/src/scraper/scrapeURL/engines/cache/index.tsindex cdff5134c..49acbcd7b 100644--- a/apps/api/src/scraper/scrapeURL/engines/cache/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/cache/index.ts@@ -4,7 +4,7 @@ import { Meta } from "../..";import { EngineError, IndexMissError } from "../../error";export async function scrapeCache(meta: Meta): Promise<EngineScrapeResult> {- const key = cacheKey(meta.url, meta.options, meta.internalOptions);+ const key = cacheKey(meta.rewrittenUrl ?? meta.url, meta.options, meta.internalOptions);if (key === null) throw new EngineError("Scrape not eligible for caching");const entry = await getEntryFromCache(key);diff --git a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts b/apps/api/src/scraper/scrapeURL/engines/docx/index.tsindex 04b30a0a8..416e77fc4 100644--- a/apps/api/src/scraper/scrapeURL/engines/docx/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/docx/index.ts@@ -4,7 +4,7 @@ import { downloadFile } from "../utils/downloadFile";import mammoth from "mammoth";export async function scrapeDOCX(meta: Meta, timeToRun: number | undefined): Promise<EngineScrapeResult> {- const { response, tempFilePath } = await downloadFile(meta.id, meta.url, {+ const { response, tempFilePath } = await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeToRun ?? 300000),});diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.tsindex 299b0d355..c4e487b84 100644--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts@@ -17,7 +17,7 @@ export async function scrapeURLWithFetch(const timeout = timeToRun ?? 300000;const mockOptions = {- url: meta.url,+ url: meta.rewrittenUrl ?? meta.url,// irrelevantmethod: "GET",@@ -55,8 +55,8 @@ export async function scrapeURLWithFetch(} else {try {const x = await Promise.race([- undici.fetch(meta.url, {- dispatcher: await makeSecureDispatcher(meta.url),+ undici.fetch(meta.rewrittenUrl ?? meta.url, {+ dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url),redirect: "follow",headers: meta.options.headers,signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 3e8490c40..84884ec90 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -209,7 +209,7 @@ export async function scrapeURLWithFireEngineChromeCDP(const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {- url: meta.url,+ url: meta.rewrittenUrl ?? meta.url,engine: "chrome-cdp",instantReturn: true,skipTlsVerification: meta.options.skipTlsVerification,@@ -298,7 +298,7 @@ export async function scrapeURLWithFireEnginePlaywright(const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestPlaywright = {- url: meta.url,+ url: meta.rewrittenUrl ?? meta.url,engine: "playwright",instantReturn: true,@@ -359,7 +359,7 @@ export async function scrapeURLWithFireEngineTLSClient(const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestTLSClient = {- url: meta.url,+ url: meta.rewrittenUrl ?? meta.url,engine: "tlsclient",instantReturn: true,diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.tsindex fe2043fe0..bf6d1fdb0 100644--- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts@@ -87,7 +87,7 @@ export async function sendDocumentToIndex(meta: Meta, document: Document) {url: normalizedURL,url_hash: urlHash,original_url: document.metadata.sourceURL ?? meta.url,- resolved_url: document.metadata.url ?? document.metadata.sourceURL ?? meta.url,+ resolved_url: document.metadata.url ?? document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,has_screenshot: document.screenshot !== undefined && meta.featureFlags.has("screenshot"),has_screenshot_fullscreen: document.screenshot !== undefined && meta.featureFlags.has("screenshot@fullScreen"),is_mobile: meta.options.mobile,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 01ef4db53..3dd97995e 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -181,14 +181,14 @@ export async function scrapePDF("base64",);return {- url: meta.pdfPrefetch.url ?? meta.url,+ url: meta.pdfPrefetch.url ?? meta.rewrittenUrl ?? meta.url,statusCode: meta.pdfPrefetch.status,html: content,markdown: content,};} else {- const file = await fetchFileToBuffer(meta.url, {+ const file = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,});@@ -212,7 +212,7 @@ export async function scrapePDF(const { response, tempFilePath } =meta.pdfPrefetch !== undefined && meta.pdfPrefetch !== null? { response: meta.pdfPrefetch, tempFilePath: meta.pdfPrefetch.filePath }- : await downloadFile(meta.id, meta.url, {+ : await downloadFile(meta.id, meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,});@@ -298,7 +298,7 @@ export async function scrapePDF(await unlink(tempFilePath);return {- url: response.url ?? meta.url,+ url: response.url ?? meta.rewrittenUrl ?? meta.url,statusCode: response.status,html: result?.html ?? "",markdown: result?.markdown ?? "",diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.tsindex 57ae8f6b1..13f47fdfd 100644--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts@@ -18,7 +18,7 @@ export async function scrapeURLWithPlaywright("Content-Type": "application/json",},body: {- url: meta.url,+ url: meta.rewrittenUrl ?? meta.url,wait_after_load: meta.options.waitFor,timeout,headers: meta.options.headers,@@ -48,7 +48,7 @@ export async function scrapeURLWithPlaywright(}return {- url: meta.url, // TODO: impove redirect following+ url: meta.rewrittenUrl ?? meta.url, // TODO: impove redirect followinghtml: response.content,statusCode: response.pageStatusCode,error: response.pageError,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 7fc043b70..73f9ebb2c 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -48,6 +48,7 @@ export type ScrapeUrlResponse = (export type Meta = {id: string;url: string;+ rewrittenUrl?: string;options: ScrapeOptions;internalOptions: InternalOptions;logger: Logger;@@ -156,9 +157,18 @@ async function buildMetaObject(});const logs: any[] = [];+ let rewrittenUrl: string | undefined;+ if (url.startsWith("https://docs.google.com/document/d/") || url.startsWith("http://docs.google.com/document/d/")) {+ const id = url.match(/\/document\/d\/([-\w]+)/)?.[1];+ if (id) {+ rewrittenUrl = `https://docs.google.com/document/d/${id}/export?format=pdf`;+ }+ }+return {id,url,+ rewrittenUrl,options,internalOptions,logger,@@ -233,7 +243,7 @@ function safeguardCircularError<T>(error: T): T {}async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {- meta.logger.info(`Scraping URL ${JSON.stringify(meta.url)}...`);+ meta.logger.info(`Scraping URL ${JSON.stringify(meta.rewrittenUrl ?? meta.url)}...`);// TODO: handle sitemap data, see WebScraper/index.ts:280// TODO: ScrapeEvents@@ -441,6 +451,11 @@ export async function scrapeURL(costTracking: CostTracking,): Promise<ScrapeUrlResponse> {const meta = await buildMetaObject(id, url, options, internalOptions, costTracking);++ if (meta.rewrittenUrl) {+ meta.logger.info("Rewriting URL");+ }+try {while (true) {try {diff --git a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.tsindex 20a6dd7ba..fee2b45d5 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractMetadata.ts@@ -12,7 +12,7 @@ export async function extractMetadataRust(return {...fromRust,...(fromRust.favicon ? {- favicon: new URL(fromRust.favicon, meta.url)+ favicon: new URL(fromRust.favicon, meta.rewrittenUrl ?? meta.url)} : {}),scrapeId: meta.id,};@@ -75,7 +75,7 @@ export async function extractMetadata(soup('link[rel*="icon"]').first().attr("href") ||undefined;if (faviconLink) {- const baseUrl = new URL(meta.url).origin;+ const baseUrl = new URL(meta.rewrittenUrl ?? meta.url).origin;favicon = faviconLink.startsWith("http")? faviconLink: `${baseUrl}${faviconLink}`;diff --git a/apps/api/src/scraper/scrapeURL/transformers/diff.ts b/apps/api/src/scraper/scrapeURL/transformers/diff.tsindex 6284d9a6f..1d313ce94 100644--- a/apps/api/src/scraper/scrapeURL/transformers/diff.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/diff.ts@@ -63,12 +63,12 @@ export async function deriveDiff(meta: Meta, document: Document): Promise<Documeconst res = await supabase_service.rpc("diff_get_last_scrape_4", {i_team_id: meta.internalOptions.teamId,- i_url: document.metadata.sourceURL ?? meta.url,+ i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,i_tag: meta.options.changeTrackingOptions?.tag ?? null,});const end = Date.now();if (end - start > 100) {- meta.logger.debug("Diffing took a while", { time: end - start, params: { i_team_id: meta.internalOptions.teamId, i_url: document.metadata.sourceURL ?? meta.url } });+ meta.logger.debug("Diffing took a while", { time: end - start, params: { i_team_id: meta.internalOptions.teamId, i_url: document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url } });}const data: {diff --git a/apps/api/src/scraper/scrapeURL/transformers/index.ts b/apps/api/src/scraper/scrapeURL/transformers/index.tsindex d91a9c36d..f229793b3 100644--- a/apps/api/src/scraper/scrapeURL/transformers/index.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/index.ts@@ -48,7 +48,7 @@ export async function deriveHTMLFromRawHTML(document.html = await htmlTransform(document.rawHtml,- document.metadata.url ?? document.metadata.sourceURL ?? meta.url,+ document.metadata.url ?? document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url,meta.options,);return document;@@ -88,7 +88,7 @@ export async function deriveLinksFromHTML(meta: Meta, document: Document): Promi);}- document.links = await extractLinks(document.html, meta.url);+ document.links = await extractLinks(document.html, document.metadata.url ?? document.metadata.sourceURL ?? meta.rewrittenUrl ?? meta.url);}return document;diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 53fe0ea71..4ca536965 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -684,7 +684,7 @@ export async function performLLMExtract(const { extractedDataArray, warning, costLimitExceededTokenUsage } =await extractData({extractOptions: generationOptions,- urls: [meta.url],+ urls: [meta.rewrittenUrl ?? meta.url],useAgent: false,scrapeId: meta.id,});@@ -760,7 +760,7 @@ export async function performLLMExtract(// // if (shouldUseSmartscrape && smartscrape_prompt) {// // meta.logger.info("Triggering SmartScrape refinement...", { reason: smartscrape_reasoning, prompt: smartscrape_prompt });// // // Call the smartScrape function (which needs to be implemented/imported)- // // // const smartScrapedDocs = await smartScrape(meta.url, smartscrape_prompt);+ // // // const smartScrapedDocs = await smartScrape(meta.rewrittenUrl ?? meta.url, smartscrape_prompt);// // // Process/merge smartScrapedDocs with extractedData// // // ... potentially update finalExtract ...// // } else {