Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Merge pull request #2329 from firecrawl/devin/ENG-3639-1759244281" (#2332)

Micah Stairs0981bee19cd2
diff --git a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
index 83ca415aa..c60b12f4c 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-viewport.test.ts
@@ -40,9 +40,9 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Screenshot Viewport", () => {
);
test(
- "should reject fullPage=true with viewport",
+ "should take full page screenshot with custom viewport width",
async () => {
- const response = await scrapeRaw(
+ const data = await scrape(
{
url: "https://example.com",
formats: [
@@ -59,11 +59,8 @@ describeIf(TEST_PRODUCTION)("V2 Scrape Screenshot Viewport", () => {
identity,
);
- expect(response.status).toBe(400);
- expect(response.body.success).toBe(false);
- expect(response.body.error).toContain(
- "Cannot specify viewport dimensions when fullPage is true",
- );
+ expect(data).toBeDefined();
+ expect(data.screenshot).toBeDefined();
},
scrapeTimeout,
);
diff --git a/apps/api/src/controllers/v2/f-search.ts b/apps/api/src/controllers/v2/f-search.ts
index 80bc74b25..67b56a2ee 100644
--- a/apps/api/src/controllers/v2/f-search.ts
+++ b/apps/api/src/controllers/v2/f-search.ts
@@ -13,10 +13,7 @@ const searchRequestSchema = z.object({
query: z.string().min(1).max(500),
limit: z.number().int().min(1).max(100).optional().default(50),
offset: z.number().int().min(0).optional().default(0),
- mode: z
- .enum(["hybrid", "keyword", "semantic", "bm25"])
- .optional()
- .default("hybrid"),
+ mode: z.enum(["hybrid", "keyword", "semantic", "bm25"]).optional().default("hybrid"),
filters: z
.object({
domain: z.string().optional(),
@@ -74,7 +71,7 @@ export async function realtimeSearchController(
// Get search index client
const client = getSearchIndexClient();
-
+
if (!client) {
res.status(503).json({
success: false,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 20204b496..dd800fecf 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -264,22 +264,17 @@ const actionSchema = z.union([
selector: z.string(),
all: z.boolean().default(false),
}),
- z
- .object({
- type: z.literal("screenshot"),
- fullPage: z.boolean().default(false),
- quality: z.number().min(1).max(100).optional(),
- viewport: z
- .object({
- width: z.number().int().positive().finite().max(7680), // 8K resolution width
- height: z.number().int().positive().finite().max(4320), // 8K resolution height
- })
- .optional(),
- })
- .refine(data => !(data.fullPage === true && data.viewport !== undefined), {
- message: "Cannot specify viewport dimensions when fullPage is true",
- path: ["viewport"],
- }),
+ z.object({
+ type: z.literal("screenshot"),
+ fullPage: z.boolean().default(false),
+ quality: z.number().min(1).max(100).optional(),
+ viewport: z
+ .object({
+ width: z.number().int().positive().finite().max(7680), // 8K resolution width
+ height: z.number().int().positive().finite().max(4320), // 8K resolution height
+ })
+ .optional(),
+ }),
z.object({
type: z.literal("write"),
text: z.string(),
@@ -370,22 +365,17 @@ type ChangeTrackingFormatWithOptions = z.output<
typeof changeTrackingFormatWithOptions
>;
-const screenshotFormatWithOptions = z
- .object({
- type: z.literal("screenshot"),
- fullPage: z.boolean().default(false),
- quality: z.number().min(1).max(100).optional(),
- viewport: z
- .object({
- width: z.number().int().positive().finite().max(7680), // 8K resolution width
- height: z.number().int().positive().finite().max(4320), // 8K resolution height
- })
- .optional(),
- })
- .refine(data => !(data.fullPage === true && data.viewport !== undefined), {
- message: "Cannot specify viewport dimensions when fullPage is true",
- path: ["viewport"],
- });
+const screenshotFormatWithOptions = z.object({
+ type: z.literal("screenshot"),
+ fullPage: z.boolean().default(false),
+ quality: z.number().min(1).max(100).optional(),
+ viewport: z
+ .object({
+ width: z.number().int().positive().finite().max(7680), // 8K resolution width
+ height: z.number().int().positive().finite().max(4320), // 8K resolution height
+ })
+ .optional(),
+});
type ScreenshotFormatWithOptions = z.output<typeof screenshotFormatWithOptions>;
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 6bc0875f2..0f393edbc 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -49,8 +49,14 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
});
}
-export async function recordRobotsBlocked(crawlId: string, url: string) {
- await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);
+export async function recordRobotsBlocked(
+ crawlId: string,
+ url: string,
+) {
+ await redisEvictConnection.sadd(
+ "crawl:" + crawlId + ":robots_blocked",
+ url,
+ );
await redisEvictConnection.expire(
"crawl:" + crawlId + ":robots_blocked",
24 * 60 * 60,
diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.ts
index ecfc43cf4..9619d1ec8 100644
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@ -135,7 +135,7 @@ export function getCategoryFromUrl(
// Check against category map for other sites
for (const [site, category] of categoryMap.entries()) {
if (site === "__pdf__") continue; // Skip the special PDF marker
-
+
if (
hostname === site.toLowerCase() ||
hostname.endsWith("." + site.toLowerCase())
diff --git a/apps/api/src/scraper/scrapeURL/engines/document/index.ts b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
index f56d23fb8..8a8cfc8f4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/document/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/document/index.ts
@@ -141,18 +141,14 @@ export async function scrapeDocument(meta: Meta): Promise<EngineScrapeResult> {
};
} finally {
// Clean up temporary file if it was created by prefetch
- if (
- tempFilePath &&
- meta.documentPrefetch !== undefined &&
- meta.documentPrefetch !== null
- ) {
+ if (tempFilePath && meta.documentPrefetch !== undefined && meta.documentPrefetch !== null) {
try {
await unlink(tempFilePath);
} catch (error) {
// Ignore errors when cleaning up temp files
- meta.logger?.warn("Failed to clean up temporary document file", {
- error,
- tempFilePath,
+ meta.logger?.warn("Failed to clean up temporary document file", {
+ error,
+ tempFilePath
});
}
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index 00e7c713e..7b300fac4 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -67,6 +67,8 @@ async function scrapePDFWithRunPodMU(
maxPages,
});
+
+
if (
process.env.PDF_MU_V2_EXPERIMENT === "true" &&
process.env.PDF_MU_V2_BASE_URL &&
@@ -113,6 +115,7 @@ async function scrapePDFWithRunPodMU(
})();
}
+
const muV1StartedAt = Date.now();
const podStart = await robustFetch({
url:
@@ -348,9 +351,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
) {
(async () => {
const startedAt = Date.now();
- const logger = meta.logger.child({
- method: "scrapePDF/MUv2Experiment",
- });
+ const logger = meta.logger.child({ method: "scrapePDF/MUv2Experiment" });
try {
const resp = await robustFetch({
url: process.env.PDF_MU_V2_BASE_URL ?? "",
diff --git a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
index 73ea508d2..549a58221 100644
--- a/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
+++ b/apps/api/src/scraper/scrapeURL/lib/extractSmartScrape.ts
@@ -209,13 +209,7 @@ const resolveRefs = (
if (refPath[0] === "#" && refPath[1] === "$defs") {
const defName = refPath[refPath.length - 1];
if (defs[defName]) {
- return resolveRefs(
- { ...defs[defName] },
- defs,
- logger,
- visited,
- depth + 1,
- );
+ return resolveRefs({ ...defs[defName] }, defs, logger, visited, depth + 1);
}
}
return obj; // Return original if ref can't be resolved
@@ -296,32 +290,24 @@ export async function extractData({
hasRefPathInString: schemaString.includes("#/$defs/"),
},
);
-
+
try {
const resolvedSchema = resolveRefs(schema, defs, logger);
const resolvedString = JSON.stringify(resolvedSchema);
- const hasRemainingRefs =
- resolvedString.includes('"$ref"') ||
- resolvedString.includes("#/$defs/");
+ const hasRemainingRefs = resolvedString.includes('"$ref"') || resolvedString.includes("#/$defs/");
if (!hasRemainingRefs) {
schema = resolvedSchema;
- if (schema && typeof schema === "object" && schema.$defs)
- delete schema.$defs;
+ if (schema && typeof schema === "object" && schema.$defs) delete schema.$defs;
logger.info("Successfully resolved schema refs", {
schema,
});
} else {
- logger.info(
- "Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema",
- );
+ logger.info("Reference resolution was skipped or incomplete (remaining $ref detected), preserving original schema");
}
} catch (error) {
- logger.warn(
- "Failed to resolve schema refs, preserving original schema",
- { error },
- );
+ logger.warn("Failed to resolve schema refs, preserving original schema", { error });
}
} else {
logger.info("No recursive references detected, resolving refs", {