Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(snips): ...
Gergő Móricz425adfbf7076
diff --git a/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts b/apps/api/src/__tests__/snips/url-validation/special-characters.test.tsdeleted file mode 100644index c91ce0078..000000000--- a/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts+++ /dev/null@@ -1,34 +0,0 @@-import { url } from "../../../controllers/v1/types";-import { describe, it, expect } from "@jest/globals";--describe("URL Schema Validation with Special Characters", () => {- it("should handle URLs with special characters in query parameters", () => {- const testUrl = "https://www.boulanger.com/c/nav-filtre/televiseur?_merchant_des~boulanger|brand~lg";-- expect(() => url.parse(testUrl)).not.toThrow();-- const parsedUrl = url.parse(testUrl);- expect(parsedUrl).toContain("_merchant_des%7Eboulanger%7Cbrand%7Elg");- });-- it("should preserve URL structure when encoding special characters", () => {- const testUrl = "https://example.com/path?param1=value1¶m2=value~with|special¶m3=normal";-- expect(() => url.parse(testUrl)).not.toThrow();-- const parsedUrl = url.parse(testUrl);- expect(parsedUrl).toContain("example.com/path?");- expect(parsedUrl).toContain("param1=value1");- expect(parsedUrl).toContain("param2=value%7Ewith%7Cspecial");- expect(parsedUrl).toContain("param3=normal");- });-- it("should handle URLs with already encoded special characters", () => {- const testUrl = "https://example.com/path?param=value%7Eencoded";-- expect(() => url.parse(testUrl)).not.toThrow();-- const parsedUrl = url.parse(testUrl);- expect(parsedUrl).toContain("param=value%7Eencoded");- });-});diff --git a/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts b/apps/api/src/__tests__/snips/v2/iframe-selectors.test.tsindex ad88ca51a..29169b710 100644--- a/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts+++ b/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from '@jest/globals';-import { scrapeOptions } from '../../../controllers/v1/types';+import { scrapeOptions } from '../../../controllers/v2/types';describe('iframe selector transformation', () => {describe('Zod schema transformation', () => {diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex 7ddd9a410..f2d5a14d3 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -1,6 +1,7 @@import { ScrapeRequestInput, ScrapeResponse, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, CrawlResponse, CrawlStatusResponse, OngoingCrawlsResponse, ErrorResponse, CrawlErrorsResponse, MapRequestInput, MapResponse, BatchScrapeRequestInput, SearchRequestInput } from "../../../controllers/v2/types";import request from "supertest";import { TEST_URL, scrapeTimeout, indexCooldown, Identity, IdmuxRequest, idmux } from "../lib";+import { SearchV2Response } from "../../../lib/entities";// Re-export shared utilities for backwards compatibilityexport { TEST_URL, scrapeTimeout, indexCooldown, Identity, IdmuxRequest, idmux };@@ -276,11 +277,9 @@ function expectSearchToSucceed(response: Awaited<ReturnType<typeof searchRaw>>)expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);expect(typeof response.body.data).toBe("object");- expect(Array.isArray(response.body.data)).toBe(true);- expect(response.body.data.length).toBeGreaterThan(0);}-export async function search(body: SearchRequestInput, identity: Identity): Promise<Document[]> {+export async function search(body: SearchRequestInput, identity: Identity): Promise<SearchV2Response> {const raw = await searchRaw(body, identity);expectSearchToSucceed(raw);return raw.body.data;diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex 1a081a52f..80cac94f1 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -12,9 +12,11 @@ beforeAll(async () => {describe("Search tests", () => {it.concurrent("works", async () => {- await search({+ const res = await search({query: "firecrawl"}, identity);+ expect(res.web).toBeDefined();+ expect(res.web?.length).toBeGreaterThan(0);}, 60000);it.concurrent("works with scrape", async () => {@@ -27,7 +29,7 @@ describe("Search tests", () => {timeout: 120000,}, identity);- for (const doc of res) {+ for (const doc of res.web ?? []) {expect(doc.markdown).toBeDefined();}}, 125000);diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsnew file mode 100644index 000000000..307efe4b0--- /dev/null+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -0,0 +1,133 @@+import { Response } from "express";+import {+ CrawlErrorsResponse,+ CrawlStatusParams,+ RequestWithAuth,+} from "./types";+import {+ getCrawl,+ getCrawlJobs,+} from "../../lib/crawl-redis";+import { getScrapeQueue } from "../../services/queue-service";+import { redisEvictConnection } from "../../../src/services/redis";+import { configDotenv } from "dotenv";+import { Job } from "bullmq";+import { supabase_rr_service } from "../../services/supabase";+import { logger } from "../../lib/logger";+configDotenv();++export async function getJob(id: string) {+ const job = await getScrapeQueue().getJob(id);+ if (!job) return job;++ return job;+}++export async function getJobs(ids: string[]) {+ const jobs: (Job & { id: string })[] = (+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))+ ).filter((x) => x) as (Job & { id: string })[];++ return jobs;+}++export async function crawlErrorsController(+ req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,+ res: Response<CrawlErrorsResponse>,+) {+ const sc = await getCrawl(req.params.jobId);++ if (sc) {+ if (sc.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }++ let jobStatuses = await Promise.all(+ (await getCrawlJobs(req.params.jobId)).map(+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ ),+ );++ const failedJobIDs: string[] = [];++ for (const [id, status] of jobStatuses) {+ if (status === "failed") {+ failedJobIDs.push(id);+ }+ }++ res.status(200).json({+ errors: (await getJobs(failedJobIDs)).map((x) => ({+ id: x.id,+ timestamp:+ x.finishedOn !== undefined+ ? new Date(x.finishedOn).toISOString()+ : undefined,+ url: x.data.url,+ error: x.failedReason,+ })),+ robotsBlocked: await redisEvictConnection.smembers(+ "crawl:" + req.params.jobId + ":robots_blocked",+ ),+ });+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {+ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("job_id", req.params.jobId)+ .limit(1)+ .throwOnError();++ if (crawlJobError) {+ logger.error("Error getting crawl job", { error: crawlJobError });+ throw crawlJobError;+ }++ const crawlJob = crawlJobs[0];++ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }++ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;++ if (+ crawlJob+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs+ ) {+ return res.status(404).json({ success: false, error: "Job expired" });+ }++ if (!crawlJobs || crawlJobs.length === 0) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }++ const { data: failedJobs, error: failedJobsError } = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("crawl_id", req.params.jobId)+ .eq("team_id", req.auth.team_id)+ .eq("success", false)+ .throwOnError();++ if (failedJobsError) {+ logger.error("Error getting failed jobs", { error: failedJobsError });+ throw failedJobsError;+ }++ res.status(200).json({+ errors: (failedJobs || []).map((job) => ({+ id: job.job_id,+ timestamp: new Date(job.date_added).toISOString(),+ url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL",+ error: job.message || "Unknown error",+ })),+ robotsBlocked: await redisEvictConnection.smembers(+ "crawl:" + req.params.jobId + ":robots_blocked",+ ),+ });+ } else {+ return res.status(404).json({ success: false, error: "Job not found" });+ }+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 8942ea93a..bb58caf24 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -917,6 +917,8 @@ export type TeamFlags = {allowZDR?: boolean;zdrCost?: number;checkRobotsOnScrape?: boolean;+ allowTeammateInvites?: boolean;+ crawlTtlHours?: number;} | null;export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;