Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(scrape-v2): Implement skipTlsVerification support for fetch and playwright engines (#1911)

Gergő Móriczb0727d3e660a
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.ts
index 577a0ee38..8969dffd2 100644
--- a/apps/api/src/__tests__/snips/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/scrape.test.ts
@@ -21,22 +21,6 @@ beforeAll(async () => {
}, 10000 + scrapeTimeout);
describe("Scrape tests", () => {
- it.concurrent("mocking works properly", async () => {
- // depends on falsified mock mocking-works-properly
- // this test will fail if mock is bypassed with real data -- firecrawl.dev will never have
- // that as its actual markdown output
-
- const response = await scrape({
- url: "http://firecrawl.dev",
- useMock: "mocking-works-properly",
- timeout: scrapeTimeout,
- }, identity);
-
- expect(response.markdown).toBe(
- "this is fake data coming from the mocking system!",
- );
- }, scrapeTimeout);
-
it.concurrent("works", async () => {
const response = await scrape({
url: "http://firecrawl.dev",
diff --git a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
index 33b18547b..cce6c8e6d 100644
--- a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
+++ b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
@@ -1,5 +1,5 @@
import request from "supertest";
-import { Identity, idmux } from "./lib";
+import { Identity, idmux, scrapeTimeout } from "./lib";
const TEST_URL = process.env.TEST_URL || "http://127.0.0.1:3002";
@@ -21,14 +21,18 @@ describe("V2 Scrape skipTlsVerification Default", () => {
.set("Content-Type", "application/json")
.send({
url: "https://expired.badssl.com/",
- timeout: 30000,
+ timeout: scrapeTimeout,
});
+
+ if (response.status !== 200) {
+ console.warn('Non-200 response:', JSON.stringify(response.body));
+ }
expect(response.status).toBe(200);
expect(response.body.success).toBe(true);
expect(response.body.data).toBeDefined();
expect(response.body.data.markdown).toContain("badssl.com");
- }, 60000);
+ }, scrapeTimeout);
test("should allow explicit skipTlsVerification: false override", async () => {
const response = await request(TEST_URL)
@@ -38,12 +42,16 @@ describe("V2 Scrape skipTlsVerification Default", () => {
.send({
url: "https://expired.badssl.com/",
skipTlsVerification: false,
- timeout: 30000,
+ timeout: scrapeTimeout,
});
+ if (response.status !== 500) {
+ console.warn('Non-500 response:', JSON.stringify(response.body));
+ }
+
expect(response.status).toBe(500);
expect(response.body.success).toBe(false);
- }, 60000);
+ }, scrapeTimeout);
test("should work with valid HTTPS sites regardless of skipTlsVerification setting", async () => {
const response = await request(TEST_URL)
@@ -52,48 +60,62 @@ describe("V2 Scrape skipTlsVerification Default", () => {
.set("Content-Type", "application/json")
.send({
url: "https://example.com",
- timeout: 30000,
+ timeout: scrapeTimeout,
});
+ if (response.status !== 200) {
+ console.warn('Non-200 response:', JSON.stringify(response.body));
+ }
+
expect(response.status).toBe(200);
expect(response.body.success).toBe(true);
expect(response.body.data).toBeDefined();
expect(response.body.data.markdown).toContain("Example Domain");
- }, 60000);
+ }, scrapeTimeout);
- test("should support object screenshot format", async () => {
- const response = await request(TEST_URL)
- .post("/v2/scrape")
- .set("Authorization", `Bearer ${identity.apiKey}`)
- .set("Content-Type", "application/json")
- .send({
- url: "https://example.com",
- formats: [{ type: "screenshot", fullPage: false }],
- timeout: 30000,
- });
+ if (!process.env.TEST_SUITE_SELF_HOSTED) {
+ test("should support object screenshot format", async () => {
+ const response = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url: "https://example.com",
+ formats: [{ type: "screenshot", fullPage: false }],
+ timeout: scrapeTimeout,
+ });
- expect(response.status).toBe(200);
- expect(response.body.success).toBe(true);
- expect(response.body.data).toBeDefined();
- expect(response.body.data.screenshot).toBeDefined();
- expect(typeof response.body.data.screenshot).toBe("string");
- }, 60000);
+ if (response.status !== 200) {
+ console.warn('Non-200 response:', JSON.stringify(response.body));
+ }
- test("should support object screenshot format with fullPage", async () => {
- const response = await request(TEST_URL)
- .post("/v2/scrape")
- .set("Authorization", `Bearer ${identity.apiKey}`)
- .set("Content-Type", "application/json")
- .send({
- url: "https://example.com",
- formats: [{ type: "screenshot", fullPage: true }],
- timeout: 30000,
- });
+ expect(response.status).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.data).toBeDefined();
+ expect(response.body.data.screenshot).toBeDefined();
+ expect(typeof response.body.data.screenshot).toBe("string");
+ }, scrapeTimeout);
- expect(response.status).toBe(200);
- expect(response.body.success).toBe(true);
- expect(response.body.data).toBeDefined();
- expect(response.body.data.screenshot).toBeDefined();
- expect(typeof response.body.data.screenshot).toBe("string");
- }, 60000);
+ test("should support object screenshot format with fullPage", async () => {
+ const response = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url: "https://example.com",
+ formats: [{ type: "screenshot", fullPage: true }],
+ timeout: scrapeTimeout,
+ });
+
+ if (response.status !== 200) {
+ console.warn('Non-200 response:', JSON.stringify(response.body));
+ }
+
+ expect(response.status).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.data).toBeDefined();
+ expect(response.body.data.screenshot).toBeDefined();
+ expect(typeof response.body.data.screenshot).toBe("string");
+ }, scrapeTimeout);
+ }
});
diff --git a/apps/api/src/__tests__/snips/v2-scrape-viewport.test.ts b/apps/api/src/__tests__/snips/v2-scrape-viewport.test.ts
index 65bdbf133..0eef69763 100644
--- a/apps/api/src/__tests__/snips/v2-scrape-viewport.test.ts
+++ b/apps/api/src/__tests__/snips/v2-scrape-viewport.test.ts
@@ -1,5 +1,5 @@
import request from "supertest";
-import { Identity, idmux } from "./lib";
+import { Identity, idmux, scrapeTimeout } from "./lib";
const TEST_URL = process.env.TEST_URL || "http://127.0.0.1:3002";
@@ -32,7 +32,7 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(200);
@@ -40,7 +40,7 @@ describe("V2 Scrape Screenshot Viewport", () => {
expect(response.body.data).toBeDefined();
expect(response.body.data.screenshot).toBeDefined();
expect(response.body.data.markdown).toBeDefined();
- }, 60000);
+ }, scrapeTimeout);
test("should take full page screenshot with custom viewport width", async () => {
const response = await request(TEST_URL)
@@ -59,14 +59,14 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(200);
expect(response.body.success).toBe(true);
expect(response.body.data).toBeDefined();
expect(response.body.data.screenshot).toBeDefined();
- }, 60000);
+ }, scrapeTimeout);
test("should work with screenshot format without viewport (backwards compatibility)", async () => {
const response = await request(TEST_URL)
@@ -76,14 +76,14 @@ describe("V2 Scrape Screenshot Viewport", () => {
.send({
url: "https://example.com",
formats: ["screenshot"],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(200);
expect(response.body.success).toBe(true);
expect(response.body.data).toBeDefined();
expect(response.body.data.screenshot).toBeDefined();
- }, 60000);
+ }, scrapeTimeout);
test("should work with object screenshot format without viewport", async () => {
const response = await request(TEST_URL)
@@ -98,14 +98,14 @@ describe("V2 Scrape Screenshot Viewport", () => {
fullPage: false
}
],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(200);
expect(response.body.success).toBe(true);
expect(response.body.data).toBeDefined();
expect(response.body.data.screenshot).toBeDefined();
- }, 60000);
+ }, scrapeTimeout);
test("should reject invalid viewport dimensions", async () => {
const response = await request(TEST_URL)
@@ -123,12 +123,12 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(400);
expect(response.body.success).toBe(false);
- }, 60000);
+ }, scrapeTimeout);
test("should reject non-integer viewport dimensions", async () => {
const response = await request(TEST_URL)
@@ -146,12 +146,12 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(400);
expect(response.body.success).toBe(false);
- }, 60000);
+ }, scrapeTimeout);
test("should reject viewport dimensions exceeding maximum limits", async () => {
const response = await request(TEST_URL)
@@ -169,12 +169,12 @@ describe("V2 Scrape Screenshot Viewport", () => {
}
}
],
- timeout: 30000,
+ timeout: scrapeTimeout,
});
expect(response.status).toBe(400);
expect(response.body.success).toBe(false);
- }, 60000);
+ }, scrapeTimeout);
} else {
it("mocked", () => {
expect(true).toBe(true);
diff --git a/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts b/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts
index 0252e44f3..7d6fed06d 100644
--- a/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts
+++ b/apps/api/src/__tests__/snips/v2-system-prompt-rejection.test.ts
@@ -1,5 +1,5 @@
import request from "supertest";
-import { Identity, idmux } from "./lib";
+import { Identity, idmux, scrapeTimeout } from "./lib";
import { ScrapeRequestInput } from "../../controllers/v2/types";
const TEST_URL = "http://127.0.0.1:3002";
@@ -38,6 +38,7 @@ describe("V2 System Prompt Rejection", () => {
systemPrompt: "Custom system prompt that should be rejected",
},
],
+ timeout: scrapeTimeout + 30000,
} as any,
identity,
);
@@ -45,7 +46,7 @@ describe("V2 System Prompt Rejection", () => {
expect(response.statusCode).toBe(400);
expect(response.body.success).toBe(false);
expect(response.body.error).toBe("Bad Request");
- });
+ }, scrapeTimeout);
it("should accept valid json format options without systemPrompt", async () => {
const response = await scrapeV2Raw(
@@ -61,13 +62,14 @@ describe("V2 System Prompt Rejection", () => {
prompt: "Extract the title",
},
],
- } as any,
+ timeout: scrapeTimeout + 30000,
+ },
identity,
);
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
- });
+ }, scrapeTimeout + 30000);
} else {
it("mocked", () => {
expect(true).toBe(true);
diff --git a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
index 1e4897cea..4ea598526 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fetch/index.ts
@@ -1,7 +1,7 @@
import * as undici from "undici";
import { EngineScrapeResult } from "..";
import { Meta } from "../..";
-import { TimeoutError } from "../../error";
+import { SSLError, TimeoutError } from "../../error";
import { specialtyScrapeCheck } from "../utils/specialtyHandler";
import {
InsecureConnectionError,
@@ -56,7 +56,7 @@ export async function scrapeURLWithFetch(
try {
const x = await Promise.race([
undici.fetch(meta.rewrittenUrl ?? meta.url, {
- dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url),
+ dispatcher: await makeSecureDispatcher(meta.rewrittenUrl ?? meta.url, meta.options.skipTlsVerification),
redirect: "follow",
headers: meta.options.headers,
signal: meta.internalOptions.abort ?? AbortSignal.timeout(timeout),
@@ -102,6 +102,8 @@ export async function scrapeURLWithFetch(
error.cause instanceof InsecureConnectionError
) {
throw error.cause;
+ } else if (error instanceof Error && error.message === "fetch failed" && error.cause && (error.cause as any).code === "CERT_HAS_EXPIRED") {
+ throw new SSLError(meta.options.skipTlsVerification);
} else {
throw error;
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 385f9aea7..797b7494f 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -310,7 +310,7 @@ export const engineOptions: {
atsv: false,
location: false,
mobile: false,
- skipTlsVerification: false,
+ skipTlsVerification: true,
useFastMode: false,
stealthProxy: false,
disableAdblock: false,
@@ -364,7 +364,7 @@ export const engineOptions: {
atsv: false,
location: false,
mobile: false,
- skipTlsVerification: false,
+ skipTlsVerification: true,
useFastMode: true,
stealthProxy: false,
disableAdblock: false,
diff --git a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
index c38bc0457..98285543b 100644
--- a/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/playwright/index.ts
@@ -22,6 +22,7 @@ export async function scrapeURLWithPlaywright(
wait_after_load: meta.options.waitFor,
timeout,
headers: meta.options.headers,
+ skip_tls_verification: meta.options.skipTlsVerification,
},
method: "POST",
logger: meta.logger.child("scrapeURLWithPlaywright/robustFetch"),
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index 258c1fc49..afaac24ae 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -44,12 +44,10 @@ function isIPv6Private(ipv6) {
export function makeSecureDispatcher(
url: string,
+ skipTlsVerification: boolean = false,
options?: undici.Agent.Options,
) {
const agentOpts: undici.Agent.Options = {
- connect: {
- rejectUnauthorized: false, // bypass SSL failures -- this is fine
- },
maxRedirections: 5000,
...options,
};
@@ -60,9 +58,17 @@ export function makeSecureDispatcher(
token: process.env.PROXY_USERNAME
? `Basic ${Buffer.from(process.env.PROXY_USERNAME + ":" + (process.env.PROXY_PASSWORD ?? "")).toString("base64")}`
: undefined,
+ requestTls: {
+ rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
+ },
...agentOpts,
})
- : new undici.Agent(agentOpts);
+ : new undici.Agent({
+ connect: {
+ rejectUnauthorized: !skipTlsVerification, // Only bypass SSL verification if explicitly requested
+ },
+ ...agentOpts,
+ });
const cookieJar = new CookieJar();
diff --git a/apps/playwright-service-ts/api.ts b/apps/playwright-service-ts/api.ts
index edbc43968..3041d8ca3 100644
--- a/apps/playwright-service-ts/api.ts
+++ b/apps/playwright-service-ts/api.ts
@@ -40,10 +40,10 @@ interface UrlModel {
timeout?: number;
headers?: { [key: string]: string };
check_selector?: string;
+ skip_tls_verification?: boolean;
}
let browser: Browser;
-let context: BrowserContext;
const initializeBrowser = async () => {
browser = await chromium.launch({
@@ -59,13 +59,16 @@ const initializeBrowser = async () => {
'--disable-gpu'
]
});
+};
+const createContext = async (skipTlsVerification: boolean = false) => {
const userAgent = new UserAgent().toString();
const viewport = { width: 1280, height: 800 };
const contextOptions: any = {
userAgent,
viewport,
+ ignoreHTTPSErrors: skipTlsVerification,
};
if (PROXY_SERVER && PROXY_USERNAME && PROXY_PASSWORD) {
@@ -80,16 +83,16 @@ const initializeBrowser = async () => {
};
}
- context = await browser.newContext(contextOptions);
+ const newContext = await browser.newContext(contextOptions);
if (BLOCK_MEDIA) {
- await context.route('**/*.{png,jpg,jpeg,gif,svg,mp3,mp4,avi,flac,ogg,wav,webm}', async (route: Route, request: PlaywrightRequest) => {
+ await newContext.route('**/*.{png,jpg,jpeg,gif,svg,mp3,mp4,avi,flac,ogg,wav,webm}', async (route: Route, request: PlaywrightRequest) => {
await route.abort();
});
}
// Intercept all requests to avoid loading ads
- await context.route('**/*', (route: Route, request: PlaywrightRequest) => {
+ await newContext.route('**/*', (route: Route, request: PlaywrightRequest) => {
const requestUrl = new URL(request.url());
const hostname = requestUrl.hostname;
@@ -99,12 +102,11 @@ const initializeBrowser = async () => {
}
return route.continue();
});
+
+ return newContext;
};
const shutdownBrowser = async () => {
- if (context) {
- await context.close();
- }
if (browser) {
await browser.close();
}
@@ -155,12 +157,14 @@ const scrapePage = async (page: Page, url: string, waitUntil: 'load' | 'networki
app.get('/health', async (req: Request, res: Response) => {
try {
- if (!browser || !context) {
+ if (!browser) {
await initializeBrowser();
}
- const testPage = await context.newPage();
+ const testContext = await createContext();
+ const testPage = await testContext.newPage();
await testPage.close();
+ await testContext.close();
res.status(200).json({ status: 'healthy' });
} catch (error) {
@@ -173,7 +177,7 @@ app.get('/health', async (req: Request, res: Response) => {
});
app.post('/scrape', async (req: Request, res: Response) => {
- const { url, wait_after_load = 0, timeout = 15000, headers, check_selector }: UrlModel = req.body;
+ const { url, wait_after_load = 0, timeout = 15000, headers, check_selector, skip_tls_verification = false }: UrlModel = req.body;
console.log(`================= Scrape Request =================`);
console.log(`URL: ${url}`);
@@ -181,6 +185,7 @@ app.post('/scrape', async (req: Request, res: Response) => {
console.log(`Timeout: ${timeout}`);
console.log(`Headers: ${headers ? JSON.stringify(headers) : 'None'}`);
console.log(`Check Selector: ${check_selector ? check_selector : 'None'}`);
+ console.log(`Skip TLS Verification: ${skip_tls_verification}`);
console.log(`==================================================`);
if (!url) {
@@ -195,11 +200,12 @@ app.post('/scrape', async (req: Request, res: Response) => {
console.warn('⚠️ WARNING: No proxy server provided. Your IP address may be blocked.');
}
- if (!browser || !context) {
+ if (!browser) {
await initializeBrowser();
}
- const page = await context.newPage();
+ const requestContext = await createContext(skip_tls_verification);
+ const page = await requestContext.newPage();
// Set headers if provided
if (headers) {
@@ -218,6 +224,7 @@ app.post('/scrape', async (req: Request, res: Response) => {
result = await scrapePage(page, url, 'networkidle', wait_after_load, timeout, check_selector);
} catch (finalError) {
await page.close();
+ await requestContext.close();
return res.status(500).json({ error: 'An error occurred while fetching the page.' });
}
}
@@ -231,6 +238,7 @@ app.post('/scrape', async (req: Request, res: Response) => {
}
await page.close();
+ await requestContext.close();
res.json({
content: result.content,