Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Merge pull request #2329 from firecrawl/devin/ENG-3639-1759244281" (#2332)
Micah Stairs0981bee19cd2
diff --git a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.tsindex 83ca415aa..c60b12f4c 100644--- a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts@@ -40,9 +40,9 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Screenshot Viewport", () => {);test(- "should reject fullPage=true with viewport",+ "should take full page screenshot with custom viewport width",async () => {- const response = await scrapeRaw(+ const data = await scrape({url: "https://example.com",formats: [@@ -59,11 +59,8 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Screenshot Viewport", () => {identity,);- expect(response.status).toBe(400);- expect(response.body.success).toBe(false);- expect(response.body.error).toContain(- "Cannot specify viewport dimensions when fullPage is true",- );+ expect(data).toBeDefined();+ expect(data.screenshot).toBeDefined();},scrapeTimeout,);diff --git a/apps/api/src/controllers/v2/f-search.ts b/apps/api/src/controllers/v2/f-search.tsindex 80bc74b25..67b56a2ee 100644--- a/apps/api/src/controllers/v2/f-search.ts+++ b/apps/api/src/controllers/v2/f-search.ts@@ -13,10 +13,7 @@ const searchRequestSchema = z.object({query: z.string().min(1).max(500),limit: z.number().int().min(1).max(100).optional().default(50),offset: z.number().int().min(0).optional().default(0),- mode: z- .enum(["hybrid", "keyword", "semantic", "bm25"])- .optional()- .default("hybrid"),+ mode: z.enum(["hybrid", "keyword", "semantic", "bm25"]).optional().default("hybrid"),filters: z.object({domain: z.string().optional(),@@ -74,7 +71,7 @@ export async function realtimeSearchController(// Get search index clientconst client = getSearchIndexClient();-+if (!client) {res.status(503).json({success: false,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 20204b496..dd800fecf 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -264,22 +264,17 @@ const actionSchema = z.union([selector: z.string(),all: z.boolean().default(false),}),- z- .object({- type: z.literal("screenshot"),- fullPage: z.boolean().default(false),- quality: z.number().min(1).max(100).optional(),- viewport: z- .object({- width: z.number().int().positive().finite().max(7680), // 8K resolution width- height: z.number().int().positive().finite().max(4320), // 8K resolution height- })- .optional(),- })- .refine(data => !(data.fullPage === true && data.viewport !== undefined), {- message: "Cannot specify viewport dimensions when fullPage is true",- path: ["viewport"],- }),+ z.object({+ type: z.literal("screenshot"),+ fullPage: z.boolean().default(false),+ quality: z.number().min(1).max(100).optional(),+ viewport: z+ .object({+ width: z.number().int().positive().finite().max(7680), // 8K resolution width+ height: z.number().int().positive().finite().max(4320), // 8K resolution height+ })+ .optional(),+ }),z.object({type: z.literal("write"),text: z.string(),@@ -370,22 +365,17 @@ type ChangeTrackingFormatWithOptions = z.output<typeof changeTrackingFormatWithOptions>;-const screenshotFormatWithOptions = z- .object({- type: z.literal("screenshot"),- fullPage: z.boolean().default(false),- quality: z.number().min(1).max(100).optional(),- viewport: z- .object({- width: z.number().int().positive().finite().max(7680), // 8K resolution width- height: z.number().int().positive().finite().max(4320), // 8K resolution height- })- .optional(),- })- .refine(data => !(data.fullPage === true && data.viewport !== undefined), {- message: "Cannot specify viewport dimensions when fullPage is true",- path: ["viewport"],- });+const screenshotFormatWithOptions = z.object({+ type: z.literal("screenshot"),+ fullPage: z.boolean().default(false),+ quality: z.number().min(1).max(100).optional(),+ viewport: z+ .object({+ width: z.number().int().positive().finite().max(7680), // 8K resolution width+ height: z.number().int().positive().finite().max(4320), // 8K resolution height+ })+ .optional(),+});type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 6bc0875f2..0f393edbc 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -49,8 +49,14 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {});}-export async function recordRobotsBlocked(crawlId: string, url: string) {- await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);+export async function recordRobotsBlocked(+ crawlId: string,+ url: string,+) {+ await redisEvictConnection.sadd(+ "crawl:" + crawlId + ":robots_blocked",+ url,+ );await redisEvictConnection.expire("crawl:" + crawlId + ":robots_blocked",24 * 60 * 60,diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex ecfc43cf4..9619d1ec8 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -135,7 +135,7 @@ export function getCategoryFromUrl(// Check against category map for other sitesfor (const [site, category] of categoryMap.entries()) {if (site === "__pdf__") continue; // Skip the special PDF marker-+if (hostname === site.toLowerCase() ||hostname.endsWith("." + site.toLowerCase())diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.tsindex f56d23fb8..8a8cfc8f4 100644--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts@@ -141,18 +141,14 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {};} finally {// Clean up temporary file if it was created by prefetch- if (- tempFilePath &&- meta.documentPrefetch !== undefined &&- meta.documentPrefetch !== null- ) {+ if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {try {await unlink(tempFilePath);} catch (error) {// Ignore errors when cleaning up temp files- meta.logger?.warn("Failed to clean up temporary document file", {- error,- tempFilePath,+ meta.logger?.warn("Failed to clean up temporary document file", {+ error,+ tempFilePath});}}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex 00e7c713e..7b300fac4 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -67,6 +67,8 @@ async function scrapePDFWithRunPodMU(maxPages,});++if (process.env.PDF_MU_V2_EXPERIMENT === "true" &&process.env.PDF_MU_V2_BASE_URL &&@@ -113,6 +115,7 @@ async function scrapePDFWithRunPodMU(})();}+const muV1StartedAt = Date.now();const podStart = await robustFetch({url:@@ -348,9 +351,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {) {(async () => {const startedAt = Date.now();- const logger = meta.logger.child({- method: "scrapePDF/MUv2Experiment",- });+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });try {const resp = await robustFetch({url: process.env.PDF_MU_V2_BASE_URL ?? "",diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.tsindex 73ea508d2..549a58221 100644--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts@@ -209,13 +209,7 @@ const resolveRefs = (if (refPath[0] === "#" && refPath[1] === "$defs") {const defName = refPath[refPath.length - 1];if (defs[defName]) {- return resolveRefs(- { ...defs[defName] },- defs,- logger,- visited,- depth + 1,- );+ return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);}}return obj; // Return original if ref can't be resolved@@ -296,32 +290,24 @@ export async function extractData({hasRefPathInString: schemaString.includes("#/$defs/"),},);-+try {const resolvedSchema = resolveRefs(schema, defs, logger);const resolvedString = JSON.stringify(resolvedSchema);- const hasRemainingRefs =- resolvedString.includes('"$ref"') ||- resolvedString.includes("#/$defs/");+ const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");if (!hasRemainingRefs) {schema = resolvedSchema;- if (schema && typeof schema === "object" && schema.$defs)- delete schema.$defs;+ if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;logger.info("Successfully resolved schema refs", {schema,});} else {- logger.info(- "Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema",- );+ logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");}} catch (error) {- logger.warn(- "Failed to resolve schema refs, preserving original schema",- { error },- );+ logger.warn("Failed to resolve schema refs, preserving original schema", { error });}} else {logger.info("No recursive references detected, resolving refs", {