Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: change skipTlsVerification default to true in v2 API (#1853)

devin-ai-integration[bot]e1e5f57453d6
diff --git a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
new file mode 100644
index 000000000..df264c92c
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts
@@ -0,0 +1,54 @@
+import request from "supertest";
+
+const TEST_URL = process.env.TEST_URL || "http://127.0.0.1:3002";
+
+describe("V2 Scrape skipTlsVerification Default", () => {
+ test("should default skipTlsVerification to true in v2 API", async () => {
+ const response = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url: "https://expired.badssl.com/",
+ timeout: 30000,
+ });
+
+ expect(response.status).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.data).toBeDefined();
+ expect(response.body.data.markdown).toContain("badssl.com");
+ }, 60000);
+
+ test("should allow explicit skipTlsVerification: false override", async () => {
+ const response = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url: "https://expired.badssl.com/",
+ skipTlsVerification: false,
+ timeout: 30000,
+ });
+
+ expect(response.status).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.data).toBeDefined();
+ expect(response.body.data.metadata.pageStatusCode).toBe(500);
+ }, 60000);
+
+ test("should work with valid HTTPS sites regardless of skipTlsVerification setting", async () => {
+ const response = await request(TEST_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send({
+ url: "https://example.com",
+ timeout: 30000,
+ });
+
+ expect(response.status).toBe(200);
+ expect(response.body.success).toBe(true);
+ expect(response.body.data).toBeDefined();
+ expect(response.body.data.markdown).toContain("Example Domain");
+ }, 60000);
+});
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
new file mode 100644
index 000000000..254bc53a9
--- /dev/null
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -0,0 +1,138 @@
+import { Response } from "express";
+import { logger as _logger } from "../../lib/logger";
+import {
+ Document,
+ RequestWithAuth,
+ ScrapeRequest,
+ scrapeRequestSchema,
+ ScrapeResponse,
+} from "./types";
+import { v4 as uuidv4 } from "uuid";
+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
+import { getJobPriority } from "../../lib/job-priority";
+import { getScrapeQueue } from "../../services/queue-service";
+
+export async function scrapeController(
+ req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
+ res: Response<ScrapeResponse>,
+) {
+ const jobId = uuidv4();
+ const preNormalizedBody = { ...req.body };
+
+ if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {
+ return res.status(400).json({
+ success: false,
+ error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",
+ });
+ }
+
+ const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;
+
+ const logger = _logger.child({
+ method: "scrapeController",
+ jobId,
+ scrapeId: jobId,
+ teamId: req.auth.team_id,
+ team_id: req.auth.team_id,
+ zeroDataRetention,
+ });
+
+ logger.debug("Scrape " + jobId + " starting", {
+ scrapeId: jobId,
+ request: req.body,
+ originalRequest: preNormalizedBody,
+ account: req.account,
+ });
+
+ req.body = scrapeRequestSchema.parse(req.body);
+
+ const origin = req.body.origin;
+ const timeout = req.body.timeout;
+
+ const startTime = new Date().getTime();
+ const jobPriority = await getJobPriority({
+ team_id: req.auth.team_id,
+ basePriority: 10,
+ });
+
+ const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;
+
+ await addScrapeJob(
+ {
+ url: req.body.url,
+ mode: "single_urls",
+ team_id: req.auth.team_id,
+ scrapeOptions: {
+ ...req.body,
+ ...(req.body.__experimental_cache ? {
+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours
+ } : {}),
+ },
+ internalOptions: {
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
+ unnormalizedSourceURL: preNormalizedBody.url,
+ bypassBilling: isDirectToBullMQ,
+ zeroDataRetention,
+ teamFlags: req.acuc?.flags ?? null,
+ },
+ origin,
+ integration: req.body.integration,
+ startTime,
+ zeroDataRetention,
+ },
+ {},
+ jobId,
+ jobPriority,
+ isDirectToBullMQ,
+ );
+
+ const totalWait =
+ (req.body.waitFor ?? 0) +
+ (req.body.actions ?? []).reduce(
+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,
+ 0,
+ );
+
+ let doc: Document;
+ try {
+ doc = await waitForJob(jobId, timeout + totalWait);
+ } catch (e) {
+ logger.error(`Error in scrapeController`, {
+ startTime,
+ });
+
+ if (zeroDataRetention) {
+ await getScrapeQueue().remove(jobId);
+ }
+
+ if (
+ e instanceof Error &&
+ (e.message.startsWith("Job wait") || e.message === "timeout")
+ ) {
+ return res.status(408).json({
+ success: false,
+ error: "Request timed out",
+ });
+ } else {
+ return res.status(500).json({
+ success: false,
+ error: `(Internal server error) - ${e && e.message ? e.message : e}`,
+ });
+ }
+ }
+
+ await getScrapeQueue().remove(jobId);
+
+ if (!req.body.formats.includes("rawHtml")) {
+ if (doc && doc.rawHtml) {
+ delete doc.rawHtml;
+ }
+ }
+
+ return res.status(200).json({
+ success: true,
+ data: doc,
+ scrape_id: origin?.includes("website") ? jobId : undefined,
+ });
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 83f11779b..eec01910d 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -332,7 +332,7 @@ const baseScrapeOptions = z
languages: z.string().array().optional(),
})
.optional(),
- skipTlsVerification: z.boolean().default(false),
+ skipTlsVerification: z.boolean().default(true),
removeBase64Images: z.boolean().default(true),
fastMode: z.boolean().default(false),
useMock: z.string().optional(),
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 13bc53787..df58a671d 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -2,6 +2,7 @@ import express from "express";
import { RateLimiterMode } from "../types";
import expressWs from "express-ws";
import { searchController } from "../controllers/v2/search";
+import { scrapeController } from "../controllers/v2/scrape";
import { batchScrapeController } from "../controllers/v2/batch-scrape";
import { crawlStatusController } from "../controllers/v2/crawl-status";
import {
@@ -21,6 +22,13 @@ v2Router.post(
wrap(searchController),
);
+v2Router.post(
+ "/scrape",
+ authMiddleware(RateLimiterMode.Scrape),
+ checkCreditsMiddleware(),
+ wrap(scrapeController),
+);
+
v2Router.post(
"/batch/scrape",
authMiddleware(RateLimiterMode.Scrape),