Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): stabilize self-hosted CI — v2 cancel owner check + query test gating (#3767)
Gergő Móricz3367ffa2868a
diff --git a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts b/apps/api/src/__tests__/snips/v2/scrape-query.test.tsindex 22b0356d7..6bf0e3d87 100644--- a/apps/api/src/__tests__/snips/v2/scrape-query.test.ts+++ b/apps/api/src/__tests__/snips/v2/scrape-query.test.ts@@ -1,4 +1,11 @@-import { concurrentIf, HAS_AI, HAS_FIREWORKS, TEST_PRODUCTION } from "../lib";+import {+ ALLOW_TEST_SUITE_WEBSITE,+ concurrentIf,+ HAS_AI,+ HAS_FIREWORKS,+ TEST_PRODUCTION,+ TEST_SUITE_WEBSITE,+} from "../lib";import {scrape,scrapeRaw,@@ -19,12 +26,12 @@ beforeAll(async () => {}, 10000 + scrapeTimeout);describe("Query format", () => {- concurrentIf(TEST_PRODUCTION || HAS_AI)(+ concurrentIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("returns a non-empty answer for a valid query",async () => {const response = await scrape({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{ type: "query", prompt: "What is Firecrawl?" }],},identity,@@ -38,12 +45,12 @@ describe("Query format", () => {scrapeTimeout,);- concurrentIf(TEST_PRODUCTION || HAS_AI)(+ concurrentIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("returns a non-empty answer for a valid question",async () => {const response = await scrape({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{ type: "question", question: "What is Firecrawl?" }],},identity,@@ -57,12 +64,12 @@ describe("Query format", () => {scrapeTimeout,);- concurrentIf(TEST_PRODUCTION || HAS_AI)(+ concurrentIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("returns both answer and markdown when formats include markdown and query",async () => {const response = await scrape({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: ["markdown",{ type: "query", prompt: "What is Firecrawl?" },@@ -80,12 +87,12 @@ describe("Query format", () => {scrapeTimeout,);- concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(+ concurrentIf(TEST_PRODUCTION || (HAS_FIREWORKS && ALLOW_TEST_SUITE_WEBSITE))("returns non-empty highlights for a valid highlights query",async () => {const response = await scrape({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{ type: "highlights", query: "What is Firecrawl?" }],},identity,@@ -99,12 +106,12 @@ describe("Query format", () => {scrapeTimeout,);- concurrentIf(TEST_PRODUCTION || HAS_FIREWORKS)(+ concurrentIf(TEST_PRODUCTION || (HAS_FIREWORKS && ALLOW_TEST_SUITE_WEBSITE))("returns a direct quote answer when query mode is directQuote",async () => {const response = await scrape({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{type: "query",@@ -123,12 +130,12 @@ describe("Query format", () => {scrapeTimeout,);- concurrentIf(TEST_PRODUCTION || HAS_AI)(+ concurrentIf(TEST_PRODUCTION || (HAS_AI && ALLOW_TEST_SUITE_WEBSITE))("does not include answer field when query format is not provided",async () => {const response = await scrape({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: ["markdown"],},identity,@@ -145,7 +152,7 @@ describe("Query format", () => {const longPrompt = "a".repeat(10001);const response = await scrapeWithFailure({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{ type: "query", prompt: longPrompt }],} as any,identity,@@ -162,7 +169,7 @@ describe("Query format", () => {async () => {const response = await scrapeWithFailure({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{ type: "question", question: "a".repeat(10001) }],} as any,identity,@@ -179,7 +186,7 @@ describe("Query format", () => {async () => {const response = await scrapeWithFailure({- url: "https://firecrawl.dev",+ url: TEST_SUITE_WEBSITE,formats: [{ type: "highlights", query: "a".repeat(10001) }],} as any,identity,diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.tsindex 57566d567..86dfc5123 100644--- a/apps/api/src/controllers/v2/crawl-cancel.ts+++ b/apps/api/src/controllers/v2/crawl-cancel.ts@@ -9,7 +9,7 @@ import {import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { RequestWithAuth, scrapeOptions } from "./types";-import { crawlGroup } from "../../services/worker/nuq";+import { crawlGroup, normalizeOwnerId } from "../../services/worker/nuq";import { removeConcurrencyLimitedJobs } from "../../lib/concurrency-limit";configDotenv();@@ -23,7 +23,9 @@ export async function crawlCancelController(return res.status(404).json({ error: "Job not found" });}- if (group.ownerId !== req.auth.team_id) {+ // group.ownerId is normalized to a UUID in NuQ, so the raw team_id+ // (e.g. "bypass" when self-hosted) must be normalized before comparing+ if (group.ownerId !== normalizeOwnerId(req.auth.team_id)) {return res.status(404).json({ error: "Job not found" });}@@ -32,12 +34,12 @@ export async function crawlCancelController(}const sc: StoredCrawl = (await getCrawl(req.params.jobId)) ?? {- team_id: group.ownerId,+ team_id: req.auth.team_id,createdAt: Date.now(),crawlerOptions: null,scrapeOptions: scrapeOptions.parse({}),internalOptions: {- teamId: group.ownerId,+ teamId: req.auth.team_id,},};diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 86465f06d..f891103af 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -55,7 +55,9 @@ type NuQOptions = {// owner IDs can sometimes be non-UUID, so let's normalize it to avoid query breakage - mogeryconst normalizedUUIDNamespace = "0f38e00e-d7ee-4b77-8a7a-a787a3537ca2";-function normalizeOwnerId(ownerId: string | undefined | null): string | null {+export function normalizeOwnerId(+ ownerId: string | undefined | null,+): string | null {if (typeof ownerId !== "string") return null;if (isUUID(ownerId)) return ownerId;return uuidv5(ownerId, normalizedUUIDNamespace);