Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: adds support for mobile web scraping
Nicolasc00cd21308d0
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 633bbdf16..c8eec815f 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -107,6 +107,7 @@ export const scrapeOptions = z.object({timeout: z.number().int().positive().finite().safe().default(30000),waitFor: z.number().int().nonnegative().finite().safe().default(0),extract: extractOptions.optional(),+ mobile: z.boolean().default(false),parsePDF: z.boolean().default(true),actions: actionsSchema.optional(),// New@@ -459,7 +460,8 @@ export function legacyScrapeOptions(x: ScrapeOptions): PageOptions {parsePDF: x.parsePDF,actions: x.actions as Action[], // no strict null checking grrrr - mogerygeolocation: x.location ?? x.geolocation,- skipTlsVerification: x.skipTlsVerification+ skipTlsVerification: x.skipTlsVerification,+ mobile: x.mobile,};}diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 8aa1d004a..0b0792d42 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -55,6 +55,7 @@ export type PageOptions = {country?: string;};skipTlsVerification?: boolean;+ mobile?: boolean;};export type ExtractorOptions = {diff --git a/apps/api/src/scraper/WebScraper/index.ts b/apps/api/src/scraper/WebScraper/index.tsindex 1817a07b8..4e39e4266 100644--- a/apps/api/src/scraper/WebScraper/index.ts+++ b/apps/api/src/scraper/WebScraper/index.ts@@ -595,6 +595,7 @@ export class WebScraperDataProvider {actions: options.pageOptions?.actions ?? undefined,geolocation: options.pageOptions?.geolocation ?? undefined,skipTlsVerification: options.pageOptions?.skipTlsVerification ?? false,+ mobile: options.pageOptions?.mobile ?? false,};this.extractorOptions = options.extractorOptions ?? { mode: "markdown" };this.replaceAllPathsWithAbsolutePaths =diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.tsindex 3bbd74eb1..5c143ec4c 100644--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts@@ -28,7 +28,7 @@ export async function scrapWithFireEngine({waitFor = 0,screenshot = false,fullPageScreenshot = false,- pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false },+ pageOptions = { parsePDF: true, atsv: false, useFastMode: false, disableJsDom: false, geolocation: { country: "US" }, skipTlsVerification: false, mobile: false },fireEngineOptions = {},headers,options,@@ -40,7 +40,7 @@ export async function scrapWithFireEngine({waitFor?: number;screenshot?: boolean;fullPageScreenshot?: boolean;- pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean };+ pageOptions?: { scrollXPaths?: string[]; parsePDF?: boolean, atsv?: boolean, useFastMode?: boolean, disableJsDom?: boolean, geolocation?: { country?: string }, skipTlsVerification?: boolean, mobile?: boolean };fireEngineOptions?: FireEngineOptions;headers?: Record<string, string>;options?: any;@@ -115,6 +115,7 @@ export async function scrapWithFireEngine({priority,engine,instantReturn: true,+ mobile: pageOptions?.mobile ?? false,...fireEngineOptionsParam,atsv: pageOptions?.atsv ?? false,scrollXPaths: pageOptions?.scrollXPaths ?? [],diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.tsindex c7185b798..945d7c82f 100644--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@ -158,6 +158,7 @@ export async function scrapSingleUrl(actions: pageOptions.actions ?? undefined,geolocation: pageOptions.geolocation ?? undefined,skipTlsVerification: pageOptions.skipTlsVerification ?? false,+ mobile: pageOptions.mobile ?? false,}if (extractorOptions) {