Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: change skipTlsVerification default to true in v2 API (#1853)
devin-ai-integration[bot]e1e5f57453d6
diff --git a/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.tsnew file mode 100644index 000000000..df264c92c--- /dev/null+++ b/apps/api/src/__tests__/snips/v2-scrape-skip-tls.test.ts@@ -0,0 +1,54 @@+import request from "supertest";++const TEST_URL = process.env.TEST_URL || "http://127.0.0.1:3002";++describe("V2 Scrape skipTlsVerification Default", () => {+ test("should default skipTlsVerification to true in v2 API", async () => {+ const response = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Content-Type", "application/json")+ .send({+ url: "https://expired.badssl.com/",+ timeout: 30000,+ });++ expect(response.status).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.data).toBeDefined();+ expect(response.body.data.markdown).toContain("badssl.com");+ }, 60000);++ test("should allow explicit skipTlsVerification: false override", async () => {+ const response = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Content-Type", "application/json")+ .send({+ url: "https://expired.badssl.com/",+ skipTlsVerification: false,+ timeout: 30000,+ });++ expect(response.status).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.data).toBeDefined();+ expect(response.body.data.metadata.pageStatusCode).toBe(500);+ }, 60000);++ test("should work with valid HTTPS sites regardless of skipTlsVerification setting", async () => {+ const response = await request(TEST_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .set("Content-Type", "application/json")+ .send({+ url: "https://example.com",+ timeout: 30000,+ });++ expect(response.status).toBe(200);+ expect(response.body.success).toBe(true);+ expect(response.body.data).toBeDefined();+ expect(response.body.data.markdown).toContain("Example Domain");+ }, 60000);+});diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsnew file mode 100644index 000000000..254bc53a9--- /dev/null+++ b/apps/api/src/controllers/v2/scrape.ts@@ -0,0 +1,138 @@+import { Response } from "express";+import { logger as _logger } from "../../lib/logger";+import {+ Document,+ RequestWithAuth,+ ScrapeRequest,+ scrapeRequestSchema,+ ScrapeResponse,+} from "./types";+import { v4 as uuidv4 } from "uuid";+import { addScrapeJob, waitForJob } from "../../services/queue-jobs";+import { getJobPriority } from "../../lib/job-priority";+import { getScrapeQueue } from "../../services/queue-service";++export async function scrapeController(+ req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,+ res: Response<ScrapeResponse>,+) {+ const jobId = uuidv4();+ const preNormalizedBody = { ...req.body };++ if (req.body.zeroDataRetention && !req.acuc?.flags?.allowZDR) {+ return res.status(400).json({+ success: false,+ error: "Zero data retention is enabled for this team. If you're interested in ZDR, please contact [email protected]",+ });+ }++ const zeroDataRetention = req.acuc?.flags?.forceZDR || req.body.zeroDataRetention;++ const logger = _logger.child({+ method: "scrapeController",+ jobId,+ scrapeId: jobId,+ teamId: req.auth.team_id,+ team_id: req.auth.team_id,+ zeroDataRetention,+ });++ logger.debug("Scrape " + jobId + " starting", {+ scrapeId: jobId,+ request: req.body,+ originalRequest: preNormalizedBody,+ account: req.account,+ });++ req.body = scrapeRequestSchema.parse(req.body);++ const origin = req.body.origin;+ const timeout = req.body.timeout;++ const startTime = new Date().getTime();+ const jobPriority = await getJobPriority({+ team_id: req.auth.team_id,+ basePriority: 10,+ });++ const isDirectToBullMQ = process.env.SEARCH_PREVIEW_TOKEN !== undefined && process.env.SEARCH_PREVIEW_TOKEN === req.body.__searchPreviewToken;++ await addScrapeJob(+ {+ url: req.body.url,+ mode: "single_urls",+ team_id: req.auth.team_id,+ scrapeOptions: {+ ...req.body,+ ...(req.body.__experimental_cache ? {+ maxAge: req.body.maxAge ?? 4 * 60 * 60 * 1000, // 4 hours+ } : {}),+ },+ internalOptions: {+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: process.env.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ unnormalizedSourceURL: preNormalizedBody.url,+ bypassBilling: isDirectToBullMQ,+ zeroDataRetention,+ teamFlags: req.acuc?.flags ?? null,+ },+ origin,+ integration: req.body.integration,+ startTime,+ zeroDataRetention,+ },+ {},+ jobId,+ jobPriority,+ isDirectToBullMQ,+ );++ const totalWait =+ (req.body.waitFor ?? 0) ++ (req.body.actions ?? []).reduce(+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 0) : 0) + a,+ 0,+ );++ let doc: Document;+ try {+ doc = await waitForJob(jobId, timeout + totalWait);+ } catch (e) {+ logger.error(`Error in scrapeController`, {+ startTime,+ });++ if (zeroDataRetention) {+ await getScrapeQueue().remove(jobId);+ }++ if (+ e instanceof Error &&+ (e.message.startsWith("Job wait") || e.message === "timeout")+ ) {+ return res.status(408).json({+ success: false,+ error: "Request timed out",+ });+ } else {+ return res.status(500).json({+ success: false,+ error: `(Internal server error) - ${e && e.message ? e.message : e}`,+ });+ }+ }++ await getScrapeQueue().remove(jobId);++ if (!req.body.formats.includes("rawHtml")) {+ if (doc && doc.rawHtml) {+ delete doc.rawHtml;+ }+ }++ return res.status(200).json({+ success: true,+ data: doc,+ scrape_id: origin?.includes("website") ? jobId : undefined,+ });+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 83f11779b..eec01910d 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -332,7 +332,7 @@ const baseScrapeOptions = zlanguages: z.string().array().optional(),}).optional(),- skipTlsVerification: z.boolean().default(false),+ skipTlsVerification: z.boolean().default(true),removeBase64Images: z.boolean().default(true),fastMode: z.boolean().default(false),useMock: z.string().optional(),diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 13bc53787..df58a671d 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -2,6 +2,7 @@ import express from "express";import { RateLimiterMode } from "../types";import expressWs from "express-ws";import { searchController } from "../controllers/v2/search";+import { scrapeController } from "../controllers/v2/scrape";import { batchScrapeController } from "../controllers/v2/batch-scrape";import { crawlStatusController } from "../controllers/v2/crawl-status";import {@@ -21,6 +22,13 @@ v2Router.post(wrap(searchController),);+v2Router.post(+ "/scrape",+ authMiddleware(RateLimiterMode.Scrape),+ checkCreditsMiddleware(),+ wrap(scrapeController),+);+v2Router.post("/batch/scrape",authMiddleware(RateLimiterMode.Scrape),