Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: Implement Scraping Excel xlsx files (#2284)

Abimael Martell8cb11c9b461a
diff --git a/apps/api/native/Cargo.toml b/apps/api/native/Cargo.toml
index 7c38f4116..37d18f12b 100644
--- a/apps/api/native/Cargo.toml
+++ b/apps/api/native/Cargo.toml
@@ -25,6 +25,7 @@ serde_json = "1.0.143"
texting_robots = "0.2.2"
url = "2.5.7"
zip = "5.0.0"
+calamine = "0.26"
[build-dependencies]
napi-build = "2"
diff --git a/apps/api/native/src/document/providers/factory.rs b/apps/api/native/src/document/providers/factory.rs
index f4ee2c4c3..9fc935bc4 100644
--- a/apps/api/native/src/document/providers/factory.rs
+++ b/apps/api/native/src/document/providers/factory.rs
@@ -2,6 +2,7 @@ use super::docx::DocxProvider;
use super::odt::OdtProvider;
use super::rtf::RtfProvider;
use super::DocumentProvider;
+use super::xlsx::XlsxProvider;
use napi_derive::napi;
#[napi]
@@ -10,12 +11,14 @@ pub enum DocumentType {
Docx,
Rtf,
Odt,
+ Xlsx,
}
pub struct ProviderFactory {
docx_provider: DocxProvider,
rtf_provider: RtfProvider,
odt_provider: OdtProvider,
+ xlsx_provider: XlsxProvider,
}
impl ProviderFactory {
@@ -24,6 +27,7 @@ impl ProviderFactory {
docx_provider: DocxProvider::new(),
rtf_provider: RtfProvider::new(),
odt_provider: OdtProvider::new(),
+ xlsx_provider: XlsxProvider::new(),
}
}
@@ -32,6 +36,7 @@ impl ProviderFactory {
DocumentType::Docx => &self.docx_provider,
DocumentType::Rtf => &self.rtf_provider,
DocumentType::Odt => &self.odt_provider,
+ DocumentType::Xlsx => &self.xlsx_provider,
}
}
}
diff --git a/apps/api/native/src/document/providers/mod.rs b/apps/api/native/src/document/providers/mod.rs
index 044b8b5c3..67899c741 100644
--- a/apps/api/native/src/document/providers/mod.rs
+++ b/apps/api/native/src/document/providers/mod.rs
@@ -5,6 +5,7 @@ pub mod docx;
pub mod factory;
pub mod odt;
pub mod rtf;
+pub mod xlsx;
pub trait DocumentProvider {
fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>>;
diff --git a/apps/api/native/src/document/providers/xlsx.rs b/apps/api/native/src/document/providers/xlsx.rs
new file mode 100644
index 000000000..896cb1ae1
--- /dev/null
+++ b/apps/api/native/src/document/providers/xlsx.rs
@@ -0,0 +1,87 @@
+use crate::document::model::*;
+use crate::document::providers::DocumentProvider;
+use calamine::{open_workbook_auto_from_rs, Data, Reader};
+use std::error::Error;
+use std::io::Cursor;
+use std::num::NonZeroU32;
+
+const ONE: NonZeroU32 = unsafe { NonZeroU32::new_unchecked(1) };
+
+pub struct XlsxProvider;
+
+impl XlsxProvider {
+ pub fn new() -> Self {
+ Self
+ }
+}
+
+impl DocumentProvider for XlsxProvider {
+ fn parse_buffer(&self, data: &[u8]) -> Result<Document, Box<dyn Error + Send + Sync>> {
+ let cursor = Cursor::new(data);
+ let mut workbook = open_workbook_auto_from_rs(cursor)?;
+
+ let mut blocks: Vec<Block> = Vec::new();
+
+ for sheet_name in workbook.sheet_names() {
+ // Add sheet heading
+ blocks.push(Block::Paragraph(Paragraph {
+ kind: ParagraphKind::Heading(2),
+ inlines: vec![Inline::Text(sheet_name.clone())],
+ }));
+
+ if let Ok(range) = workbook.worksheet_range(&sheet_name) {
+ let mut rows: Vec<TableRow> = Vec::new();
+ for r in range.rows() {
+ let mut cells: Vec<TableCell> = Vec::new();
+ for cell in r {
+ let text = data_type_to_string(cell);
+ let blocks_in_cell = if text.trim().is_empty() {
+ Vec::new()
+ } else {
+ vec![Block::Paragraph(Paragraph {
+ kind: ParagraphKind::Normal,
+ inlines: vec![Inline::Text(text)],
+ })]
+ };
+ cells.push(TableCell {
+ blocks: blocks_in_cell,
+ colspan: ONE,
+ rowspan: ONE,
+ });
+ }
+ rows.push(TableRow {
+ cells,
+ kind: TableRowKind::Body,
+ });
+ }
+
+ blocks.push(Block::Table(Table { rows }));
+ }
+ }
+
+ Ok(Document {
+ blocks,
+ metadata: DocumentMetadata::default(),
+ notes: Vec::new(),
+ comments: Vec::new(),
+ })
+ }
+
+ fn name(&self) -> &'static str {
+ "xlsx"
+ }
+}
+
+fn data_type_to_string(cell: &Data) -> String {
+ match cell {
+ Data::Empty => String::new(),
+ Data::String(s) => s.clone(),
+ Data::Float(f) => f.to_string(),
+ Data::Int(i) => i.to_string(),
+ Data::Bool(b) => b.to_string(),
+ Data::DateTime(v) => v.to_string(),
+ Data::DateTimeIso(v) => v.to_string(),
+ Data::DurationIso(v) => v.to_string(),
+ Data::Error(e) => format!("#ERROR({e:?})"),
+ }
+}
diff --git a/apps/api/samples/sample.xlsx b/apps/api/samples/sample.xlsx
new file mode 100644
index 000000000..a578d22a7
Binary files /dev/null and b/apps/api/samples/sample.xlsx differ
diff --git a/apps/api/src/__tests__/snips/v2/document-converter.test.ts b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
index c8d0d6d15..3ed176fec 100644
--- a/apps/api/src/__tests__/snips/v2/document-converter.test.ts
+++ b/apps/api/src/__tests__/snips/v2/document-converter.test.ts
@@ -43,4 +43,18 @@ describe("Document Converter tests", () => {
);
},
);
+
+ describe("XLSX document conversion", () => {
+ const expectedHtml = `<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Document</title></head><body><main><h2>Sheet1</h2><table><tbody><tr><td>sample file</td><td>test</td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>iPhone</td><td>1000</td></tr><tr><td>iPad</td><td>800</td></tr><tr><td>Macbook</td><td>1200</td></tr></tbody></table><h2>Sheet2</h2><table><tbody><tr><td>other tab</td><td></td></tr><tr><td></td><td></td></tr><tr><td>Name</td><td>Price</td></tr><tr><td>ChatGPT</td><td>20</td></tr><tr><td>Claude</td><td>17</td></tr><tr><td>Perplexity</td><td>20</td></tr></tbody></table></main></body></html>`;
+
+ it("should convert XLSX document and return expected HTML", async () => {
+ const filePath = path.join(samplesDir, "sample.xlsx");
+ const fileBuffer = fs.readFileSync(filePath);
+ const html = await converter.convertBufferToHtml(
+ new Uint8Array(fileBuffer),
+ DocumentType.Xlsx,
+ );
+ expect(html).toBe(expectedHtml);
+ });
+ });
});
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index fabe2e04d..004909c99 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -11,10 +11,12 @@ import {
DNSResolutionError,
UnsupportedFileError,
PDFAntibotError,
+ DocumentAntibotError,
PDFInsufficientTimeError,
NoEnginesLeftError,
ZDRViolationError,
PDFPrefetchFailed,
+ DocumentPrefetchFailed,
SiteError,
SSLError,
ProxySelectionError,
@@ -30,10 +32,12 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_SITE_ERROR: SiteError,
SCRAPE_PROXY_SELECTION_ERROR: ProxySelectionError,
SCRAPE_PDF_PREFETCH_FAILED: PDFPrefetchFailed,
+ SCRAPE_DOCUMENT_PREFETCH_FAILED: DocumentPrefetchFailed,
SCRAPE_ZDR_VIOLATION_ERROR: ZDRViolationError,
SCRAPE_DNS_RESOLUTION_ERROR: DNSResolutionError,
SCRAPE_PDF_INSUFFICIENT_TIME_ERROR: PDFInsufficientTimeError,
SCRAPE_PDF_ANTIBOT_ERROR: PDFAntibotError,
+ SCRAPE_DOCUMENT_ANTIBOT_ERROR: DocumentAntibotError,
SCRAPE_UNSUPPORTED_FILE_ERROR: UnsupportedFileError,
SCRAPE_ACTION_ERROR: ActionError,
SCRAPE_RACED_REDIRECT_ERROR: RacedRedirectError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 7fdd0847c..05378a775 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -7,10 +7,12 @@ export type ErrorCodes =
| "SCRAPE_SITE_ERROR"
| "SCRAPE_PROXY_SELECTION_ERROR"
| "SCRAPE_PDF_PREFETCH_FAILED"
+ | "SCRAPE_DOCUMENT_PREFETCH_FAILED"
| "SCRAPE_ZDR_VIOLATION_ERROR"
| "SCRAPE_DNS_RESOLUTION_ERROR"
| "SCRAPE_PDF_INSUFFICIENT_TIME_ERROR"
| "SCRAPE_PDF_ANTIBOT_ERROR"
+ | "SCRAPE_DOCUMENT_ANTIBOT_ERROR"
| "SCRAPE_UNSUPPORTED_FILE_ERROR"
| "SCRAPE_ACTION_ERROR"
| "SCRAPE_RACED_REDIRECT_ERROR"
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index 649b520c0..8a8cfc8f4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -2,14 +2,29 @@ import { Meta } from "../..";
import { EngineScrapeResult } from "..";
import { fetchFileToBuffer } from "../utils/downloadFile";
import { DocumentConverter, DocumentType } from "@mendable/firecrawl-rs";
+import type { Response } from "undici";
+import { DocumentAntibotError, DocumentPrefetchFailed } from "../../error";
+import { readFile, unlink } from "node:fs/promises";
const converter = new DocumentConverter();
function getDocumentTypeFromUrl(url: string): DocumentType {
const urlLower = url.toLowerCase();
- if (urlLower.endsWith(".docx")) return DocumentType.Docx;
- if (urlLower.endsWith(".odt")) return DocumentType.Odt;
- if (urlLower.endsWith(".rtf")) return DocumentType.Rtf;
+
+ // Check for extensions at the end or in the middle (e.g., file.xlsx/hash)
+ if (urlLower.endsWith(".docx") || urlLower.includes(".docx/"))
+ return DocumentType.Docx;
+ if (urlLower.endsWith(".odt") || urlLower.includes(".odt/"))
+ return DocumentType.Odt;
+ if (urlLower.endsWith(".rtf") || urlLower.includes(".rtf/"))
+ return DocumentType.Rtf;
+ if (
+ urlLower.endsWith(".xlsx") ||
+ urlLower.endsWith(".xls") ||
+ urlLower.includes(".xlsx/") ||
+ urlLower.includes(".xls/")
+ )
+ return DocumentType.Xlsx;
return DocumentType.Docx; // hope for the best
}
@@ -38,33 +53,106 @@ function getDocumentTypeFromContentType(
return DocumentType.Rtf;
}
+ if (
+ ct.includes(
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ ) ||
+ ct.includes("application/vnd.ms-excel")
+ ) {
+ return DocumentType.Xlsx;
+ }
+
return null;
}
+function isValidDocumentContentType(contentType: string | null): boolean {
+ if (!contentType) return false;
+
+ const ct = contentType.toLowerCase();
+ const validTypes = [
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-excel",
+ "application/msword",
+ "application/rtf",
+ "text/rtf",
+ "application/vnd.oasis.opendocument.text",
+ ];
+
+ return validTypes.some(type => ct.includes(type));
+}
+
export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
- const { response, buffer } = await fetchFileToBuffer(
- meta.rewrittenUrl ?? meta.url,
- {
+ let response: Response;
+ let buffer: Buffer;
+ let proxyUsed: "basic" | "stealth" = "basic";
+ let tempFilePath: string | null = null;
+
+ if (meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {
+ // Use prefetched document
+ tempFilePath = meta.documentPrefetch.filePath;
+ buffer = await readFile(tempFilePath);
+
+ // Create a mock response object with content-type from prefetch
+ const headers = new Headers();
+ if (meta.documentPrefetch.contentType) {
+ headers.set("Content-Type", meta.documentPrefetch.contentType);
+ }
+
+ response = {
+ url: meta.documentPrefetch.url ?? meta.rewrittenUrl ?? meta.url,
+ status: meta.documentPrefetch.status,
+ headers,
+ } as Response;
+
+ proxyUsed = meta.documentPrefetch.proxyUsed;
+ } else {
+ // Fetch the document normally
+ const result = await fetchFileToBuffer(meta.rewrittenUrl ?? meta.url, {
headers: meta.options.headers,
signal: meta.abort.asSignal(),
- },
- );
-
- const documentType =
- getDocumentTypeFromContentType(response.headers.get("content-type")) ??
- getDocumentTypeFromUrl(response.url);
-
- const html = await converter.convertBufferToHtml(
- new Uint8Array(buffer),
- documentType,
- );
-
- return {
- url: response.url,
- statusCode: response.status,
- html,
- proxyUsed: "basic",
- };
+ });
+ response = result.response;
+ buffer = result.buffer;
+
+ // Validate content type only when fetching directly (not using prefetch)
+ const ct = response.headers.get("Content-Type");
+ if (ct && !isValidDocumentContentType(ct)) {
+ // if downloaded file wasn't a valid document, throw antibot error
+ throw new DocumentAntibotError();
+ }
+ }
+
+ try {
+ const documentType =
+ getDocumentTypeFromContentType(response.headers.get("content-type")) ??
+ getDocumentTypeFromUrl(response.url);
+
+ const html = await converter.convertBufferToHtml(
+ new Uint8Array(buffer),
+ documentType,
+ );
+
+ return {
+ url: response.url,
+ statusCode: response.status,
+ html,
+ proxyUsed,
+ };
+ } finally {
+ // Clean up temporary file if it was created by prefetch
+ if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {
+ try {
+ await unlink(tempFilePath);
+ } catch (error) {
+ // Ignore errors when cleaning up temp files
+ meta.logger?.warn("Failed to clean up temporary document file", {
+ error,
+ tempFilePath
+ });
+ }
+ }
+ }
}
export function documentMaxReasonableTime(meta: Meta): number {
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
index 6a85c6af9..cb9f6c4dc 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/specialtyHandler.ts
@@ -6,18 +6,21 @@ import os from "os";
import { writeFile } from "fs/promises";
import { Meta } from "../..";
-async function feResToPdfPrefetch(
+async function feResToFilePrefetch(
logger: Logger,
feRes: FireEngineCheckStatusSuccess | undefined,
-): Promise<Meta["pdfPrefetch"]> {
+ fileExtension: string,
+ fileType: string,
+ contentType?: string,
+): Promise<Meta["pdfPrefetch"] | Meta["documentPrefetch"]> {
if (!feRes?.file) {
- logger.warn("No file in pdf prefetch");
+ logger.warn(`No file in ${fileType} prefetch`);
return null;
}
const filePath = path.join(
os.tmpdir(),
- `tempFile-${crypto.randomUUID()}.pdf`,
+ `tempFile-${crypto.randomUUID()}.${fileExtension}`,
);
await writeFile(filePath, Buffer.from(feRes.file.content, "base64"));
@@ -26,9 +29,43 @@ async function feResToPdfPrefetch(
url: feRes.url,
filePath,
proxyUsed: feRes.usedMobileProxy ? "stealth" : "basic",
+ contentType,
};
}
+async function feResToPdfPrefetch(
+ logger: Logger,
+ feRes: FireEngineCheckStatusSuccess | undefined,
+): Promise<Meta["pdfPrefetch"]> {
+ return feResToFilePrefetch(logger, feRes, "pdf", "pdf");
+}
+
+async function feResToDocumentPrefetch(
+ logger: Logger,
+ feRes: FireEngineCheckStatusSuccess | undefined,
+ contentType: string,
+): Promise<Meta["documentPrefetch"]> {
+ // Determine file extension from content type
+ let extension = "tmp";
+ if (
+ contentType.includes("wordprocessingml") ||
+ contentType.includes("msword")
+ ) {
+ extension = "docx";
+ } else if (
+ contentType.includes("spreadsheetml") ||
+ contentType.includes("ms-excel")
+ ) {
+ extension = "xlsx";
+ } else if (contentType.includes("opendocument.text")) {
+ extension = "odt";
+ } else if (contentType.includes("rtf")) {
+ extension = "rtf";
+ }
+
+ return feResToFilePrefetch(logger, feRes, extension, "document", contentType);
+}
+
export async function specialtyScrapeCheck(
logger: Logger,
headers: Record<string, string> | undefined,
@@ -45,24 +82,54 @@ export async function specialtyScrapeCheck(
return;
}
- if (
- contentType === "application/pdf" ||
- contentType.startsWith("application/pdf;") ||
- (contentType === "application/octet-stream" &&
- (feRes?.file?.content.startsWith("JVBERi0") ||
- feRes?.content.startsWith("%PDF-")))
- ) {
- throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
- }
-
const documentTypes = [
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-excel",
+ "application/msword",
"application/rtf",
"text/rtf",
"application/vnd.oasis.opendocument.text",
];
- if (documentTypes.some(type => contentType.startsWith(type))) {
- throw new AddFeatureError(["document"]);
+ const isDocument = documentTypes.some(type => contentType.startsWith(type));
+ const isPdf =
+ contentType === "application/pdf" ||
+ contentType.startsWith("application/pdf;");
+ const isOctetStream = contentType === "application/octet-stream";
+
+ // Check for document types first (before PDF to prioritize documents)
+ if (isDocument) {
+ throw new AddFeatureError(
+ ["document"],
+ undefined,
+ await feResToDocumentPrefetch(logger, feRes, contentType),
+ );
+ }
+
+ // Check for octet-stream with document signature (Office files are ZIP archives starting with "PK")
+ if (
+ isOctetStream &&
+ (feRes?.file?.content.startsWith("UEsD") || feRes?.content.startsWith("PK"))
+ ) {
+ throw new AddFeatureError(
+ ["document"],
+ undefined,
+ await feResToDocumentPrefetch(logger, feRes, contentType),
+ );
+ }
+
+ // Check for PDF
+ if (isPdf) {
+ throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
+ }
+
+ // Check for octet-stream with PDF signature
+ if (
+ isOctetStream &&
+ (feRes?.file?.content.startsWith("JVBERi0") ||
+ feRes?.content.startsWith("%PDF-"))
+ ) {
+ throw new AddFeatureError(["pdf"], await feResToPdfPrefetch(logger, feRes));
}
}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 40e3da160..77e246a55 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -41,11 +41,17 @@ export class NoEnginesLeftError extends TransportableError {
export class AddFeatureError extends Error {
public featureFlags: FeatureFlag[];
public pdfPrefetch: Meta["pdfPrefetch"];
+ public documentPrefetch: Meta["documentPrefetch"];
- constructor(featureFlags: FeatureFlag[], pdfPrefetch?: Meta["pdfPrefetch"]) {
+ constructor(
+ featureFlags: FeatureFlag[],
+ pdfPrefetch?: Meta["pdfPrefetch"],
+ documentPrefetch?: Meta["documentPrefetch"],
+ ) {
super("New feature flags have been discovered: " + featureFlags.join(", "));
this.featureFlags = featureFlags;
this.pdfPrefetch = pdfPrefetch;
+ this.documentPrefetch = documentPrefetch;
}
}
@@ -316,6 +322,51 @@ export class PDFPrefetchFailed extends TransportableError {
}
}
+export class DocumentAntibotError extends TransportableError {
+ constructor() {
+ super(
+ "SCRAPE_DOCUMENT_ANTIBOT_ERROR",
+ "Document scrape was prevented by anti-bot",
+ );
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new DocumentAntibotError();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
+export class DocumentPrefetchFailed extends TransportableError {
+ constructor() {
+ const message = isSelfHosted()
+ ? "Failed to prefetch document that is protected by anti-bot. Please check your logs for more details."
+ : "Failed to prefetch document that is protected by anti-bot. Please contact [email protected]";
+
+ super("SCRAPE_DOCUMENT_PREFETCH_FAILED", message);
+ }
+
+ serialize() {
+ return super.serialize();
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new DocumentPrefetchFailed();
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class FEPageLoadFailed extends Error {
constructor() {
super(
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 850b4660e..d3ad70226 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -27,6 +27,7 @@ import {
EngineError,
NoEnginesLeftError,
PDFAntibotError,
+ DocumentAntibotError,
RemoveFeatureError,
SiteError,
UnsupportedFileError,
@@ -36,6 +37,7 @@ import {
DNSResolutionError,
ZDRViolationError,
PDFPrefetchFailed,
+ DocumentPrefetchFailed,
FEPageLoadFailed,
EngineSnipedError,
WaterfallNextEngineSignal,
@@ -95,6 +97,17 @@ export type Meta = {
url?: string;
status: number;
proxyUsed: "basic" | "stealth";
+ contentType?: string;
+ }
+ | null
+ | undefined; // undefined: no prefetch yet, null: prefetch came back empty
+ documentPrefetch:
+ | {
+ filePath: string;
+ url?: string;
+ status: number;
+ proxyUsed: "basic" | "stealth";
+ contentType?: string;
}
| null
| undefined; // undefined: no prefetch yet, null: prefetch came back empty
@@ -152,16 +165,24 @@ function buildFeatureFlags(
const urlO = new URL(url);
const lowerPath = urlO.pathname.toLowerCase();
- if (lowerPath.endsWith(".pdf")) {
- flags.add("pdf");
- }
-
- if (
+ // Check for document types first (they take precedence over PDF)
+ const isDocument =
lowerPath.endsWith(".docx") ||
lowerPath.endsWith(".odt") ||
- lowerPath.endsWith(".rtf")
- ) {
+ lowerPath.endsWith(".rtf") ||
+ lowerPath.endsWith(".xlsx") ||
+ lowerPath.endsWith(".xls") ||
+ lowerPath.includes(".docx/") ||
+ lowerPath.includes(".odt/") ||
+ lowerPath.includes(".rtf/") ||
+ lowerPath.includes(".xlsx/") ||
+ lowerPath.includes(".xls/");
+
+ if (isDocument) {
flags.add("document");
+ } else if (lowerPath.endsWith(".pdf") || lowerPath.includes(".pdf/")) {
+ // Only add PDF flag if it's not a document
+ flags.add("pdf");
}
if (options.blockAds === false) {
@@ -277,6 +298,7 @@ async function buildMetaObject(
? await loadMock(options.useMock, _logger)
: null,
pdfPrefetch: undefined,
+ documentPrefetch: undefined,
costTracking,
};
}
@@ -562,6 +584,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof ActionError ||
error.error instanceof UnsupportedFileError ||
error.error instanceof PDFAntibotError ||
+ error.error instanceof DocumentAntibotError ||
error.error instanceof PDFInsufficientTimeError ||
error.error instanceof ProxySelectionError
) {
@@ -948,6 +971,9 @@ export async function scrapeURL(
if (error.pdfPrefetch) {
meta.pdfPrefetch = error.pdfPrefetch;
}
+ if (error.documentPrefetch) {
+ meta.documentPrefetch = error.documentPrefetch;
+ }
} else if (
error instanceof RemoveFeatureError &&
(meta.internalOptions.forceEngine === undefined ||
@@ -980,6 +1006,23 @@ export async function scrapeURL(
[...meta.featureFlags].filter(x => x !== "pdf"),
);
}
+ } else if (
+ error instanceof DocumentAntibotError &&
+ meta.internalOptions.forceEngine === undefined
+ ) {
+ if (meta.documentPrefetch !== undefined) {
+ meta.logger.error(
+ "Document was prefetched and still blocked by antibot, failing",
+ );
+ throw error;
+ } else {
+ meta.logger.debug(
+ "Document was blocked by anti-bot, prefetching with chrome-cdp",
+ );
+ meta.featureFlags = new Set(
+ [...meta.featureFlags].filter(x => x !== "document"),
+ );
+ }
} else {
throw error;
}
@@ -1062,6 +1105,12 @@ export async function scrapeURL(
"scrapeURL: Failed to prefetch PDF that is protected by anti-bot",
{ error },
);
+ } else if (error instanceof DocumentPrefetchFailed) {
+ errorType = "DocumentPrefetchFailed";
+ meta.logger.warn(
+ "scrapeURL: Failed to prefetch document that is protected by anti-bot",
+ { error },
+ );
} else if (error instanceof ProxySelectionError) {
errorType = "ProxySelectionError";
meta.logger.warn("scrapeURL: Proxy selection error", { error });
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index c379b92ab..707e23fff 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -388,6 +388,31 @@ describe("Standalone scrapeURL tests", () => {
}
}, 60000);
+ it("Scrapes a XLSX file", async () => {
+ const out = await scrapeURL(
+ "test:scrape-xlsx",
+ "https://download.microsoft.com/download/1/4/E/14EDED28-6C58-4055-A65C-23B4DA81C4DE/Financial%20Sample.xlsx",
+ scrapeOptions.parse({}),
+ { teamId: "test" },
+ new CostTracking(),
+ );
+
+ // expect(out.logs.length).toBeGreaterThan(0);
+ expect(out.success).toBe(true);
+ if (out.success) {
+ expect(out.document.warning).toBeUndefined();
+ expect(out.document).toHaveProperty("metadata");
+ // sheet name
+ expect(out.document.markdown).toContain("Sheet1");
+ // headers
+ expect(out.document.markdown).toContain("Segment");
+ expect(out.document.markdown).toContain("Product");
+ expect(out.document.markdown).toContain("Country");
+ expect(out.document.metadata.statusCode).toBe(200);
+ expect(out.document.metadata.error).toBeUndefined();
+ }
+ }, 60000);
+
it("LLM extract with prompt and schema", async () => {
const out = await scrapeURL(
"test:llm-extract-prompt-schema",