Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: Implement Scraping Excel xlsx files (#2284)
Abimael Martell8cb11c9b461a
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.tomlindex 7c38f4116..37d18f12b 100644--- a/apps/api/native/Cargo.toml+++ b/apps/api/native/Cargo.toml@@ -25,6 +25,7 @@ serde_json = "1.0.143"texting_robots = "0.2.2"url = "2.5.7"zip = "5.0.0"+calamine = "0.26"[build-dependencies]napi-build = "2"diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rsindex f4ee2c4c3..9fc935bc4 100644--- a/apps/api/native/src/document/providers/factory.rs+++ b/apps/api/native/src/document/providers/factory.rs@@ -2,6 +2,7 @@ use super::docx::DocxProvider;use super::odt::OdtProvider;use super::rtf::RtfProvider;use super::DocumentProvider;+use super::xlsx::XlsxProvider;use napi_derive::napi;#[napi]@@ -10,12 +11,14 @@ pub enum DocumentType {Docx,Rtf,Odt,+ Xlsx,}pub struct ProviderFactory {docx_provider: DocxProvider,rtf_provider: RtfProvider,odt_provider: OdtProvider,+ xlsx_provider: XlsxProvider,}impl ProviderFactory {@@ -24,6 +27,7 @@ impl ProviderFactory {docx_provider: DocxProvider::new(),rtf_provider: RtfProvider::new(),odt_provider: OdtProvider::new(),+ xlsx_provider: XlsxProvider::new(),}}@@ -32,6 +36,7 @@ impl ProviderFactory {DocumentType::Docx => &self.docx_provider,DocumentType::Rtf => &self.rtf_provider,DocumentType::Odt => &self.odt_provider,+ DocumentType::Xlsx => &self.xlsx_provider,}}}diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rsindex 044b8b5c3..67899c741 100644--- a/apps/api/native/src/document/providers/mod.rs+++ b/apps/api/native/src/document/providers/mod.rs@@ -5,6 +5,7 @@ pub mod docx;pub mod factory;pub mod odt;pub mod rtf;+pub mod xlsx;pub trait DocumentProvider {fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>>;diff --git a/apps/api/native/src/document/providers/xlsx.rs b/apps/api/native/src/document/providers/xlsx.rsnew file mode 100644index 000000000..896cb1ae1--- /dev/null+++ b/apps/api/native/src/document/providers/xlsx.rs@@ -0,0 +1,87 @@+use crate::document::model::*;+use crate::document::providers::DocumentProvider;+use calamine::{open_workbook_auto_from_rs, Data, Reader};+use std::error::Error;+use std::io::Cursor;+use std::num::NonZeroU32;++const ONE: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(1) };++pub struct XlsxProvider;++impl XlsxProvider {+ pub fn new() -> Self {+ Self+ }+}++impl DocumentProvider for XlsxProvider {+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {+ let cursor = Cursor::new(data);+ let mut workbook = open_workbook_auto_from_rs(cursor)?;++ let mut blocks: Vec<Block> = Vec::new();++ for sheet_name in workbook.sheet_names() {+ // Add sheet heading+ blocks.push(Block::Paragraph(Paragraph {+ kind: ParagraphKind::Heading(2),+ inlines: vec![Inline::Text(sheet_name.clone())],+ }));++ if let Ok(range) = workbook.worksheet_range(&sheet_name) {+ let mut rows: Vec<TableRow> = Vec::new();+ for r in range.rows() {+ let mut cells: Vec<TableCell> = Vec::new();+ for cell in r {+ let text = data_type_to_string(cell);+ let blocks_in_cell = if text.trim().is_empty() {+ Vec::new()+ } else {+ vec![Block::Paragraph(Paragraph {+ kind: ParagraphKind::Normal,+ inlines: vec![Inline::Text(text)],+ })]+ };+ cells.push(TableCell {+ blocks: blocks_in_cell,+ colspan: ONE,+ rowspan: ONE,+ });+ }+ rows.push(TableRow {+ cells,+ kind: TableRowKind::Body,+ });+ }++ blocks.push(Block::Table(Table { rows }));+ }+ }++ Ok(Document {+ blocks,+ metadata: DocumentMetadata::default(),+ notes: Vec::new(),+ comments: Vec::new(),+ })+ }++ fn name(&self) -> &'static str {+ "xlsx"+ }+}++fn data_type_to_string(cell: &Data) -> String {+ match cell {+ Data::Empty => String::new(),+ Data::String(s) => s.clone(),+ Data::Float(f) => f.to_string(),+ Data::Int(i) => i.to_string(),+ Data::Bool(b) => b.to_string(),+ Data::DateTime(v) => v.to_string(),+ Data::DateTimeIso(v) => v.to_string(),+ Data::DurationIso(v) => v.to_string(),+ Data::Error(e) => format!("#ERROR({e:?})"),+ }+}diff --git a/apps/api/samples/sample.xlsx b/apps/api/samples/sample.xlsxnew file mode 100644index 000000000..a578d22a7Binary files /dev/null and b/apps/api/samples/sample.xlsx differdiff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.tsindex c8d0d6d15..3ed176fec 100644--- a/apps/api/src/__tests__/snips/v2/document-converter.test.ts+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts@@ -43,4 +43,18 @@ describe("Document Converter tests", () => {);},);++ describe("XLSX document conversion", () => {+ const expectedHtml = `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><h2>Sheet1</h2><table><tbody><tr><td>sample file</td><td>test</td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>iPhone</td><td>1000</td></tr><tr><td>iPad</td><td>800</td></tr><tr><td>Macbook</td><td>1200</td></tr></tbody></table><h2>Sheet2</h2><table><tbody><tr><td>other tab</td><td></td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>ChatGPT</td><td>20</td></tr><tr><td>Claude</td><td>17</td></tr><tr><td>Perplexity</td><td>20</td></tr></tbody></table></main></body></html>`;++ it("should convert XLSX document and return expected HTML", async () => {+ const filePath = path.join(samplesDir, "sample.xlsx");+ const fileBuffer = fs.readFileSync(filePath);+ const html = await converter.convertBufferToHtml(+ new Uint8Array(fileBuffer),+ DocumentType.Xlsx,+ );+ expect(html).toBe(expectedHtml);+ });+ });});diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex fabe2e04d..004909c99 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -11,10 +11,12 @@ import {DNSResolutionError,UnsupportedFileError,PDFAntibotError,+ DocumentAntibotError,PDFInsufficientTimeError,NoEnginesLeftError,ZDRViolationError,PDFPrefetchFailed,+ DocumentPrefetchFailed,SiteError,SSLError,ProxySelectionError,@@ -30,10 +32,12 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_SITE_ERROR: SiteError,SCRAPE_PROXY_SELECTION_ERROR: ProxySelectionError,SCRAPE_PDF_PREFETCH_FAILED: PDFPrefetchFailed,+ SCRAPE_DOCUMENT_PREFETCH_FAILED: DocumentPrefetchFailed,SCRAPE_ZDR_VIOLATION_ERROR: ZDRViolationError,SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,+ SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,SCRAPE_ACTION_ERROR: ActionError,SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex 7fdd0847c..05378a775 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -7,10 +7,12 @@ export type ErrorCodes =| "SCRAPE_SITE_ERROR"| "SCRAPE_PROXY_SELECTION_ERROR"| "SCRAPE_PDF_PREFETCH_FAILED"+ | "SCRAPE_DOCUMENT_PREFETCH_FAILED"| "SCRAPE_ZDR_VIOLATION_ERROR"| "SCRAPE_DNS_RESOLUTION_ERROR"| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"| "SCRAPE_PDF_ANTIBOT_ERROR"+ | "SCRAPE_DOCUMENT_ANTIBOT_ERROR"| "SCRAPE_UNSUPPORTED_FILE_ERROR"| "SCRAPE_ACTION_ERROR"| "SCRAPE_RACED_REDIRECT_ERROR"diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex 649b520c0..8a8cfc8f4 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -2,14 +2,29 @@ import { Meta } from "../..";import { EngineScrapeResult } from "..";import { fetchFileToBuffer } from "../utils/downloadFile";import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";+import type { Response } from "undici";+import { DocumentAntibotError, DocumentPrefetchFailed } from "../../error";+import { readFile, unlink } from "node:fs/promises";const converter = new DocumentConverter();function getDocumentTypeFromUrl(url: string): DocumentType {const urlLower = url.toLowerCase();- if (urlLower.endsWith(".docx")) return DocumentType.Docx;- if (urlLower.endsWith(".odt")) return DocumentType.Odt;- if (urlLower.endsWith(".rtf")) return DocumentType.Rtf;++ // Check for extensions at the end or in the middle (e.g., file.xlsx/hash)+ if (urlLower.endsWith(".docx") || urlLower.includes(".docx/"))+ return DocumentType.Docx;+ if (urlLower.endsWith(".odt") || urlLower.includes(".odt/"))+ return DocumentType.Odt;+ if (urlLower.endsWith(".rtf") || urlLower.includes(".rtf/"))+ return DocumentType.Rtf;+ if (+ urlLower.endsWith(".xlsx") ||+ urlLower.endsWith(".xls") ||+ urlLower.includes(".xlsx/") ||+ urlLower.includes(".xls/")+ )+ return DocumentType.Xlsx;return DocumentType.Docx; // hope for the best}@@ -38,33 +53,106 @@ function getDocumentTypeFromContentType(return DocumentType.Rtf;}+ if (+ ct.includes(+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ ) ||+ ct.includes("application/vnd.ms-excel")+ ) {+ return DocumentType.Xlsx;+ }+return null;}+function isValidDocumentContentType(contentType: string | null): boolean {+ if (!contentType) return false;++ const ct = contentType.toLowerCase();+ const validTypes = [+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ "application/vnd.ms-excel",+ "application/msword",+ "application/rtf",+ "text/rtf",+ "application/vnd.oasis.opendocument.text",+ ];++ return validTypes.some(type => ct.includes(type));+}+export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {- const { response, buffer } = await fetchFileToBuffer(- meta.rewrittenUrl ?? meta.url,- {+ let response: Response;+ let buffer: Buffer;+ let proxyUsed: "basic" | "stealth" = "basic";+ let tempFilePath: string | null = null;++ if (meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {+ // Use prefetched document+ tempFilePath = meta.documentPrefetch.filePath;+ buffer = await readFile(tempFilePath);++ // Create a mock response object with content-type from prefetch+ const headers = new Headers();+ if (meta.documentPrefetch.contentType) {+ headers.set("Content-Type", meta.documentPrefetch.contentType);+ }++ response = {+ url: meta.documentPrefetch.url ?? meta.rewrittenUrl ?? meta.url,+ status: meta.documentPrefetch.status,+ headers,+ } as Response;++ proxyUsed = meta.documentPrefetch.proxyUsed;+ } else {+ // Fetch the document normally+ const result = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {headers: meta.options.headers,signal: meta.abort.asSignal(),- },- );-- const documentType =- getDocumentTypeFromContentType(response.headers.get("content-type")) ??- getDocumentTypeFromUrl(response.url);-- const html = await converter.convertBufferToHtml(- new Uint8Array(buffer),- documentType,- );-- return {- url: response.url,- statusCode: response.status,- html,- proxyUsed: "basic",- };+ });+ response = result.response;+ buffer = result.buffer;++ // Validate content type only when fetching directly (not using prefetch)+ const ct = response.headers.get("Content-Type");+ if (ct && !isValidDocumentContentType(ct)) {+ // if downloaded file wasn't a valid document, throw antibot error+ throw new DocumentAntibotError();+ }+ }++ try {+ const documentType =+ getDocumentTypeFromContentType(response.headers.get("content-type")) ??+ getDocumentTypeFromUrl(response.url);++ const html = await converter.convertBufferToHtml(+ new Uint8Array(buffer),+ documentType,+ );++ return {+ url: response.url,+ statusCode: response.status,+ html,+ proxyUsed,+ };+ } finally {+ // Clean up temporary file if it was created by prefetch+ if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {+ try {+ await unlink(tempFilePath);+ } catch (error) {+ // Ignore errors when cleaning up temp files+ meta.logger?.warn("Failed to clean up temporary document file", {+ error,+ tempFilePath+ });+ }+ }+ }}export function documentMaxReasonableTime(meta: Meta): number {diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.tsindex 6a85c6af9..cb9f6c4dc 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts@@ -6,18 +6,21 @@ import os from "os";import { writeFile } from "fs/promises";import { Meta } from "../..";-async function feResToPdfPrefetch(+async function feResToFilePrefetch(logger: Logger,feRes: FireEngineCheckStatusSuccess | undefined,-): Promise<Meta["pdfPrefetch"]> {+ fileExtension: string,+ fileType: string,+ contentType?: string,+): Promise<Meta["pdfPrefetch"] | Meta["documentPrefetch"]> {if (!feRes?.file) {- logger.warn("No file in pdf prefetch");+ logger.warn(`No file in ${fileType} prefetch`);return null;}const filePath = path.join(os.tmpdir(),- `tempFile-${crypto.randomUUID()}.pdf`,+ `tempFile-${crypto.randomUUID()}.${fileExtension}`,);await writeFile(filePath, Buffer.from(feRes.file.content, "base64"));@@ -26,9 +29,43 @@ async function feResToPdfPrefetch(url: feRes.url,filePath,proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",+ contentType,};}+async function feResToPdfPrefetch(+ logger: Logger,+ feRes: FireEngineCheckStatusSuccess | undefined,+): Promise<Meta["pdfPrefetch"]> {+ return feResToFilePrefetch(logger, feRes, "pdf", "pdf");+}++async function feResToDocumentPrefetch(+ logger: Logger,+ feRes: FireEngineCheckStatusSuccess | undefined,+ contentType: string,+): Promise<Meta["documentPrefetch"]> {+ // Determine file extension from content type+ let extension = "tmp";+ if (+ contentType.includes("wordprocessingml") ||+ contentType.includes("msword")+ ) {+ extension = "docx";+ } else if (+ contentType.includes("spreadsheetml") ||+ contentType.includes("ms-excel")+ ) {+ extension = "xlsx";+ } else if (contentType.includes("opendocument.text")) {+ extension = "odt";+ } else if (contentType.includes("rtf")) {+ extension = "rtf";+ }++ return feResToFilePrefetch(logger, feRes, extension, "document", contentType);+}+export async function specialtyScrapeCheck(logger: Logger,headers: Record<string, string> | undefined,@@ -45,24 +82,54 @@ export async function specialtyScrapeCheck(return;}- if (- contentType === "application/pdf" ||- contentType.startsWith("application/pdf;") ||- (contentType === "application/octet-stream" &&- (feRes?.file?.content.startsWith("JVBERi0") ||- feRes?.content.startsWith("%PDF-")))- ) {- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));- }-const documentTypes = ["application/vnd.openxmlformats-officedocument.wordprocessingml.document",+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",+ "application/vnd.ms-excel",+ "application/msword","application/rtf","text/rtf","application/vnd.oasis.opendocument.text",];- if (documentTypes.some(type => contentType.startsWith(type))) {- throw new AddFeatureError(["document"]);+ const isDocument = documentTypes.some(type => contentType.startsWith(type));+ const isPdf =+ contentType === "application/pdf" ||+ contentType.startsWith("application/pdf;");+ const isOctetStream = contentType === "application/octet-stream";++ // Check for document types first (before PDF to prioritize documents)+ if (isDocument) {+ throw new AddFeatureError(+ ["document"],+ undefined,+ await feResToDocumentPrefetch(logger, feRes, contentType),+ );+ }++ // Check for octet-stream with document signature (Office files are ZIP archives starting with "PK")+ if (+ isOctetStream &&+ (feRes?.file?.content.startsWith("UEsD") || feRes?.content.startsWith("PK"))+ ) {+ throw new AddFeatureError(+ ["document"],+ undefined,+ await feResToDocumentPrefetch(logger, feRes, contentType),+ );+ }++ // Check for PDF+ if (isPdf) {+ throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));+ }++ // Check for octet-stream with PDF signature+ if (+ isOctetStream &&+ (feRes?.file?.content.startsWith("JVBERi0") ||+ feRes?.content.startsWith("%PDF-"))+ ) {+ throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));}}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 40e3da160..77e246a55 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -41,11 +41,17 @@ export class NoEnginesLeftError extends TransportableError {export class AddFeatureError extends Error {public featureFlags: FeatureFlag[];public pdfPrefetch: Meta["pdfPrefetch"];+ public documentPrefetch: Meta["documentPrefetch"];- constructor(featureFlags: FeatureFlag[], pdfPrefetch?: Meta["pdfPrefetch"]) {+ constructor(+ featureFlags: FeatureFlag[],+ pdfPrefetch?: Meta["pdfPrefetch"],+ documentPrefetch?: Meta["documentPrefetch"],+ ) {super("New feature flags have been discovered: " + featureFlags.join(", "));this.featureFlags = featureFlags;this.pdfPrefetch = pdfPrefetch;+ this.documentPrefetch = documentPrefetch;}}@@ -316,6 +322,51 @@ export class PDFPrefetchFailed extends TransportableError {}}+export class DocumentAntibotError extends TransportableError {+ constructor() {+ super(+ "SCRAPE_DOCUMENT_ANTIBOT_ERROR",+ "Document scrape was prevented by anti-bot",+ );+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new DocumentAntibotError();+ x.stack = data.stack;+ return x;+ }+}++export class DocumentPrefetchFailed extends TransportableError {+ constructor() {+ const message = isSelfHosted()+ ? "Failed to prefetch document that is protected by anti-bot. Please check your logs for more details."+ : "Failed to prefetch document that is protected by anti-bot. Please contact [email protected]";++ super("SCRAPE_DOCUMENT_PREFETCH_FAILED", message);+ }++ serialize() {+ return super.serialize();+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new DocumentPrefetchFailed();+ x.stack = data.stack;+ return x;+ }+}+export class FEPageLoadFailed extends Error {constructor() {super(diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 850b4660e..d3ad70226 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -27,6 +27,7 @@ import {EngineError,NoEnginesLeftError,PDFAntibotError,+ DocumentAntibotError,RemoveFeatureError,SiteError,UnsupportedFileError,@@ -36,6 +37,7 @@ import {DNSResolutionError,ZDRViolationError,PDFPrefetchFailed,+ DocumentPrefetchFailed,FEPageLoadFailed,EngineSnipedError,WaterfallNextEngineSignal,@@ -95,6 +97,17 @@ export type Meta = {url?: string;status: number;proxyUsed: "basic" | "stealth";+ contentType?: string;+ }+ | null+ | undefined; // undefined: no prefetch yet, null: prefetch came back empty+ documentPrefetch:+ | {+ filePath: string;+ url?: string;+ status: number;+ proxyUsed: "basic" | "stealth";+ contentType?: string;}| null| undefined; // undefined: no prefetch yet, null: prefetch came back empty@@ -152,16 +165,24 @@ function buildFeatureFlags(const urlO = new URL(url);const lowerPath = urlO.pathname.toLowerCase();- if (lowerPath.endsWith(".pdf")) {- flags.add("pdf");- }-- if (+ // Check for document types first (they take precedence over PDF)+ const isDocument =lowerPath.endsWith(".docx") ||lowerPath.endsWith(".odt") ||- lowerPath.endsWith(".rtf")- ) {+ lowerPath.endsWith(".rtf") ||+ lowerPath.endsWith(".xlsx") ||+ lowerPath.endsWith(".xls") ||+ lowerPath.includes(".docx/") ||+ lowerPath.includes(".odt/") ||+ lowerPath.includes(".rtf/") ||+ lowerPath.includes(".xlsx/") ||+ lowerPath.includes(".xls/");++ if (isDocument) {flags.add("document");+ } else if (lowerPath.endsWith(".pdf") || lowerPath.includes(".pdf/")) {+ // Only add PDF flag if it's not a document+ flags.add("pdf");}if (options.blockAds === false) {@@ -277,6 +298,7 @@ async function buildMetaObject(? await loadMock(options.useMock, _logger): null,pdfPrefetch: undefined,+ documentPrefetch: undefined,costTracking,};}@@ -562,6 +584,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof ActionError ||error.error instanceof UnsupportedFileError ||error.error instanceof PDFAntibotError ||+ error.error instanceof DocumentAntibotError ||error.error instanceof PDFInsufficientTimeError ||error.error instanceof ProxySelectionError) {@@ -948,6 +971,9 @@ export async function scrapeURL(if (error.pdfPrefetch) {meta.pdfPrefetch = error.pdfPrefetch;}+ if (error.documentPrefetch) {+ meta.documentPrefetch = error.documentPrefetch;+ }} else if (error instanceof RemoveFeatureError &&(meta.internalOptions.forceEngine === undefined ||@@ -980,6 +1006,23 @@ export async function scrapeURL([...meta.featureFlags].filter(x => x !== "pdf"),);}+ } else if (+ error instanceof DocumentAntibotError &&+ meta.internalOptions.forceEngine === undefined+ ) {+ if (meta.documentPrefetch !== undefined) {+ meta.logger.error(+ "Document was prefetched and still blocked by antibot, failing",+ );+ throw error;+ } else {+ meta.logger.debug(+ "Document was blocked by anti-bot, prefetching with chrome-cdp",+ );+ meta.featureFlags = new Set(+ [...meta.featureFlags].filter(x => x !== "document"),+ );+ }} else {throw error;}@@ -1062,6 +1105,12 @@ export async function scrapeURL("scrapeURL: Failed to prefetch PDF that is protected by anti-bot",{ error },);+ } else if (error instanceof DocumentPrefetchFailed) {+ errorType = "DocumentPrefetchFailed";+ meta.logger.warn(+ "scrapeURL: Failed to prefetch document that is protected by anti-bot",+ { error },+ );} else if (error instanceof ProxySelectionError) {errorType = "ProxySelectionError";meta.logger.warn("scrapeURL: Proxy selection error", { error });diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex c379b92ab..707e23fff 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -388,6 +388,31 @@ describe("Standalone scrapeURL tests", () => {}}, 60000);+ it("Scrapes a XLSX file", async () => {+ const out = await scrapeURL(+ "test:scrape-xlsx",+ "https://download.microsoft.com/download/1/4/E/14EDED28-6C58-4055-A65C-23B4DA81C4DE/Financial%20Sample.xlsx",+ scrapeOptions.parse({}),+ { teamId: "test" },+ new CostTracking(),+ );++ // expect(out.logs.length).toBeGreaterThan(0);+ expect(out.success).toBe(true);+ if (out.success) {+ expect(out.document.warning).toBeUndefined();+ expect(out.document).toHaveProperty("metadata");+ // sheet name+ expect(out.document.markdown).toContain("Sheet1");+ // headers+ expect(out.document.markdown).toContain("Segment");+ expect(out.document.markdown).toContain("Product");+ expect(out.document.markdown).toContain("Country");+ expect(out.document.metadata.statusCode).toBe(200);+ expect(out.document.metadata.error).toBeUndefined();+ }+ }, 60000);+it("LLM extract with prompt and schema", async () => {const out = await scrapeURL("test:llm-extract-prompt-schema",