Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(snips): ...

Gergő Móricz425adfbf7076
diff --git a/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts b/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts
deleted file mode 100644
index c91ce0078..000000000
--- a/apps/api/src/__tests__/snips/url-validation/special-characters.test.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-import { url } from "../../../controllers/v1/types";
-import { describe, it, expect } from "@jest/globals";
-
-describe("URL Schema Validation with Special Characters", () => {
- it("should handle URLs with special characters in query parameters", () => {
- const testUrl = "https://www.boulanger.com/c/nav-filtre/televiseur?_merchant_des~boulanger|brand~lg";
-
- expect(() => url.parse(testUrl)).not.toThrow();
-
- const parsedUrl = url.parse(testUrl);
- expect(parsedUrl).toContain("_merchant_des%7Eboulanger%7Cbrand%7Elg");
- });
-
- it("should preserve URL structure when encoding special characters", () => {
- const testUrl = "https://example.com/path?param1=value1&param2=value~with|special&param3=normal";
-
- expect(() => url.parse(testUrl)).not.toThrow();
-
- const parsedUrl = url.parse(testUrl);
- expect(parsedUrl).toContain("example.com/path?");
- expect(parsedUrl).toContain("param1=value1");
- expect(parsedUrl).toContain("param2=value%7Ewith%7Cspecial");
- expect(parsedUrl).toContain("param3=normal");
- });
-
- it("should handle URLs with already encoded special characters", () => {
- const testUrl = "https://example.com/path?param=value%7Eencoded";
-
- expect(() => url.parse(testUrl)).not.toThrow();
-
- const parsedUrl = url.parse(testUrl);
- expect(parsedUrl).toContain("param=value%7Eencoded");
- });
-});
diff --git a/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts b/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts
index ad88ca51a..29169b710 100644
--- a/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts
+++ b/apps/api/src/__tests__/snips/v2/iframe-selectors.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from '@jest/globals';
-import { scrapeOptions } from '../../../controllers/v1/types';
+import { scrapeOptions } from '../../../controllers/v2/types';
describe('iframe selector transformation', () => {
describe('Zod schema transformation', () => {
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index 7ddd9a410..f2d5a14d3 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -1,6 +1,7 @@
import { ScrapeRequestInput, ScrapeResponse, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, CrawlResponse, CrawlStatusResponse, OngoingCrawlsResponse, ErrorResponse, CrawlErrorsResponse, MapRequestInput, MapResponse, BatchScrapeRequestInput, SearchRequestInput } from "../../../controllers/v2/types";
import request from "supertest";
import { TEST_URL, scrapeTimeout, indexCooldown, Identity, IdmuxRequest, idmux } from "../lib";
+import { SearchV2Response } from "../../../lib/entities";
// Re-export shared utilities for backwards compatibility
export { TEST_URL, scrapeTimeout, indexCooldown, Identity, IdmuxRequest, idmux };
@@ -276,11 +277,9 @@ function expectSearchToSucceed(response: Awaited<ReturnType<typeof searchRaw>>)
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
expect(typeof response.body.data).toBe("object");
- expect(Array.isArray(response.body.data)).toBe(true);
- expect(response.body.data.length).toBeGreaterThan(0);
}
-export async function search(body: SearchRequestInput, identity: Identity): Promise<Document[]> {
+export async function search(body: SearchRequestInput, identity: Identity): Promise<SearchV2Response> {
const raw = await searchRaw(body, identity);
expectSearchToSucceed(raw);
return raw.body.data;
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts
index 1a081a52f..80cac94f1 100644
--- a/apps/api/src/__tests__/snips/v2/search.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search.test.ts
@@ -12,9 +12,11 @@ beforeAll(async () => {
describe("Search tests", () => {
it.concurrent("works", async () => {
- await search({
+ const res = await search({
query: "firecrawl"
}, identity);
+ expect(res.web).toBeDefined();
+ expect(res.web?.length).toBeGreaterThan(0);
}, 60000);
it.concurrent("works with scrape", async () => {
@@ -27,7 +29,7 @@ describe("Search tests", () => {
timeout: 120000,
}, identity);
- for (const doc of res) {
+ for (const doc of res.web ?? []) {
expect(doc.markdown).toBeDefined();
}
}, 125000);
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
new file mode 100644
index 000000000..307efe4b0
--- /dev/null
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -0,0 +1,133 @@
+import { Response } from "express";
+import {
+ CrawlErrorsResponse,
+ CrawlStatusParams,
+ RequestWithAuth,
+} from "./types";
+import {
+ getCrawl,
+ getCrawlJobs,
+} from "../../lib/crawl-redis";
+import { getScrapeQueue } from "../../services/queue-service";
+import { redisEvictConnection } from "../../../src/services/redis";
+import { configDotenv } from "dotenv";
+import { Job } from "bullmq";
+import { supabase_rr_service } from "../../services/supabase";
+import { logger } from "../../lib/logger";
+configDotenv();
+
+export async function getJob(id: string) {
+ const job = await getScrapeQueue().getJob(id);
+ if (!job) return job;
+
+ return job;
+}
+
+export async function getJobs(ids: string[]) {
+ const jobs: (Job & { id: string })[] = (
+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
+ ).filter((x) => x) as (Job & { id: string })[];
+
+ return jobs;
+}
+
+export async function crawlErrorsController(
+ req: RequestWithAuth<CrawlStatusParams, undefined, CrawlErrorsResponse>,
+ res: Response<CrawlErrorsResponse>,
+) {
+ const sc = await getCrawl(req.params.jobId);
+
+ if (sc) {
+ if (sc.team_id !== req.auth.team_id) {
+ return res.status(403).json({ success: false, error: "Forbidden" });
+ }
+
+ let jobStatuses = await Promise.all(
+ (await getCrawlJobs(req.params.jobId)).map(
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ ),
+ );
+
+ const failedJobIDs: string[] = [];
+
+ for (const [id, status] of jobStatuses) {
+ if (status === "failed") {
+ failedJobIDs.push(id);
+ }
+ }
+
+ res.status(200).json({
+ errors: (await getJobs(failedJobIDs)).map((x) => ({
+ id: x.id,
+ timestamp:
+ x.finishedOn !== undefined
+ ? new Date(x.finishedOn).toISOString()
+ : undefined,
+ url: x.data.url,
+ error: x.failedReason,
+ })),
+ robotsBlocked: await redisEvictConnection.smembers(
+ "crawl:" + req.params.jobId + ":robots_blocked",
+ ),
+ });
+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("job_id", req.params.jobId)
+ .limit(1)
+ .throwOnError();
+
+ if (crawlJobError) {
+ logger.error("Error getting crawl job", { error: crawlJobError });
+ throw crawlJobError;
+ }
+
+ const crawlJob = crawlJobs[0];
+
+ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {
+ return res.status(403).json({ success: false, error: "Forbidden" });
+ }
+
+ const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
+ const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
+
+ if (
+ crawlJob
+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > crawlTtlMs
+ ) {
+ return res.status(404).json({ success: false, error: "Job expired" });
+ }
+
+ if (!crawlJobs || crawlJobs.length === 0) {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
+
+ const { data: failedJobs, error: failedJobsError } = await supabase_rr_service
+ .from("firecrawl_jobs")
+ .select("*")
+ .eq("crawl_id", req.params.jobId)
+ .eq("team_id", req.auth.team_id)
+ .eq("success", false)
+ .throwOnError();
+
+ if (failedJobsError) {
+ logger.error("Error getting failed jobs", { error: failedJobsError });
+ throw failedJobsError;
+ }
+
+ res.status(200).json({
+ errors: (failedJobs || []).map((job) => ({
+ id: job.job_id,
+ timestamp: new Date(job.date_added).toISOString(),
+ url: job.page_options?.url || job.page_options?.urls?.[0] || "Unknown URL",
+ error: job.message || "Unknown error",
+ })),
+ robotsBlocked: await redisEvictConnection.smembers(
+ "crawl:" + req.params.jobId + ":robots_blocked",
+ ),
+ });
+ } else {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 8942ea93a..bb58caf24 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -917,6 +917,8 @@ export type TeamFlags = {
allowZDR?: boolean;
zdrCost?: number;
checkRobotsOnScrape?: boolean;
+ allowTeammateInvites?: boolean;
+ crawlTtlHours?: number;
} | null;
export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;