Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): enforce the country check on agent and browser/interact routes (#5204)
Gergő Móricz146d3bf3eeb5
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 34004dcf5..6fa3890ea 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -300,6 +300,11 @@ jobs:run: pnpm exec vitest run src/controllers/v2/__tests__/agent-location.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run country check route regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=default+ working-directory: apps/api+- name: Install test site dependenciesrun: pnpm install --frozen-lockfileworking-directory: apps/test-sitediff --git a/apps/api/src/__tests__/routes/country-check.routes.test.ts b/apps/api/src/__tests__/routes/country-check.routes.test.tsnew file mode 100644index 000000000..91171b342--- /dev/null+++ b/apps/api/src/__tests__/routes/country-check.routes.test.ts@@ -0,0 +1,176 @@+import express from "express";+import request from "supertest";++const mocks = vi.hoisted(() => ({+ lookup: vi.fn(),+ auth: { team_id: "team-1" } as { team_id: string },+ acuc: undefined as { flags: Record<string, unknown> } | undefined,+}));++vi.mock("geoip-country", () => ({ lookup: mocks.lookup }));++vi.mock("../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../config")>();+ return {+ ...actual,+ config: {+ ...actual.config,+ USE_DB_AUTHENTICATION: true,+ RESTRICTED_COUNTRIES: ["XX"],+ },+ };+});++// Real routers and country checks; auth is stubbed and the credit check ends+// the chain with a 204, so a 204 means the request got past the country check.+vi.mock("../../routes/shared", async importOriginal => {+ const actual = await importOriginal<typeof import("../../routes/shared")>();+ return {+ ...actual,+ authMiddleware: () => (req: any, _res: any, next: any) => {+ req.auth = mocks.auth;+ req.acuc = mocks.acuc;+ next();+ },+ checkCreditsMiddleware: () => (_req: any, res: any) =>+ res.status(204).end(),+ };+});++import { config } from "../../config";+import { v2Router } from "../../routes/v2";++const app = express();+app.use(express.json());+app.use("/v2", v2Router);++const JOB_ID = "11111111-1111-4111-8111-111111111111";+const SESSION_ID = "22222222-2222-4222-8222-222222222222";++const ALWAYS_CHECKED: Array<[string, Record<string, unknown>]> = [+ ["/v2/agent", { prompt: "find the pricing page" }],+ ["/v2/agent", { prompt: "and now the docs", threadId: JOB_ID }],+ ["/v2/interact", {}],+ ["/v2/browser", {}],+ [`/v2/scrape/${JOB_ID}/interact`, { code: "console.log(1)" }],+ [`/v2/interact/${SESSION_ID}/execute`, { code: "console.log(1)" }],+ [`/v2/browser/${SESSION_ID}/execute`, { code: "console.log(1)" }],+];++function fromCountry(country: string) {+ mocks.lookup.mockReturnValue({ country });+}++beforeEach(() => {+ mocks.lookup.mockReset();+ mocks.auth = { team_id: "team-1" };+ mocks.acuc = { flags: {} };+ config.USE_DB_AUTHENTICATION = true;+});++describe("country check on agent and browser routes", () => {+ for (const [path, body] of ALWAYS_CHECKED) {+ it(`rejects POST ${path} ${JSON.stringify(body)} from a restricted country`, async () => {+ fromCountry("XX");++ const res = await request(app).post(path).send(body);++ expect(res.status).toBe(403);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain(+ "the agent, and browser/interact sessions is not allowed by default in your country",+ );+ expect(mocks.lookup).toHaveBeenCalled();+ });+ }++ it("lets agent and interact requests from other countries through", async () => {+ fromCountry("YY");++ expect(+ (await request(app).post("/v2/agent").send({ prompt: "x" })).status,+ ).toBe(204);+ expect((await request(app).post("/v2/interact").send({})).status).toBe(204);+ const interact = await request(app)+ .post("/v2/scrape/not-a-uuid/interact")+ .send({ code: "console.log(1)" });+ expect(interact.status).toBe(400);+ expect(interact.body.error).toContain("Invalid job ID format");+ });++ it("rejects keyless scrape interact without a team ACUC", async () => {+ fromCountry("XX");+ mocks.auth = { team_id: "preview_keyless_127.0.0.1" };+ mocks.acuc = undefined;++ const res = await request(app)+ .post(`/v2/scrape/${JOB_ID}/interact`)+ .send({ code: "console.log(1)" });++ expect(res.status).toBe(403);+ });++ it("honors the team's skipCountryCheck flag", async () => {+ fromCountry("XX");+ mocks.acuc = { flags: { skipCountryCheck: true } };++ expect(+ (await request(app).post("/v2/agent").send({ prompt: "x" })).status,+ ).toBe(204);+ expect((await request(app).post("/v2/browser").send({})).status).toBe(204);+ });++ it("points self-hosted deployments at their server configuration", async () => {+ fromCountry("XX");+ config.USE_DB_AUTHENTICATION = false;++ const res = await request(app).post("/v2/agent").send({ prompt: "x" });++ expect(res.status).toBe(403);+ expect(res.body.error).toContain("check your server configuration");+ });++ it("uses the hosted contact copy otherwise", async () => {+ fromCountry("XX");++ const res = await request(app).post("/v2/agent").send({ prompt: "x" });++ expect(res.body.error).toContain("[email protected]");+ });+});++describe("country check on scrape-style routes", () => {+ it("lets a plain scrape from a restricted country through", async () => {+ fromCountry("XX");++ const res = await request(app)+ .post("/v2/scrape")+ .send({ url: "https://example.com" });++ expect(res.status).toBe(204);+ expect(mocks.lookup).not.toHaveBeenCalled();+ });++ it("rejects a scrape with headers from a restricted country", async () => {+ fromCountry("XX");++ const res = await request(app)+ .post("/v2/scrape")+ .send({ url: "https://example.com", headers: { "x-test": "1" } });++ expect(res.status).toBe(403);+ });++ it("rejects a crawl with scrape actions from a restricted country", async () => {+ fromCountry("XX");++ const res = await request(app)+ .post("/v2/crawl")+ .send({+ url: "https://example.com",+ scrapeOptions: { actions: [{ type: "wait", milliseconds: 1 }] },+ });++ expect(res.status).toBe(403);+ });+});diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 21131e0fb..4f138b19d 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -487,28 +487,6 @@ export function countryCheck(return next();}- const couldBeRestricted =- req.body &&- (req.body.actions ||- (req.body.headers &&- typeof req.body.headers === "object" &&- Object.keys(req.body.headers).length > 0) ||- req.body.agent ||- req.body.jsonOptions?.agent ||- req.body.extract?.agent ||- req.body.scrapeOptions?.actions ||- (req.body.scrapeOptions?.headers &&- typeof req.body.scrapeOptions.headers === "object" &&- Object.keys(req.body.scrapeOptions.headers).length > 0) ||- req.body.scrapeOptions?.agent ||- req.body.scrapeOptions?.jsonOptions?.agent ||- req.body.scrapeOptions?.extract?.agent ||- req.path.startsWith("/v2/agent"));-- if (!couldBeRestricted) {- return next();- }-if (!req.ip) {logger.warn("IP address not found, unable to check country");return next();@@ -529,14 +507,43 @@ export function countryCheck(return res.status(403).json({success: false,error: isSelfHosted()- ? "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please check your server configuration."- : "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please contact us at [email protected]",+ ? "Use of headers, actions, the agent, and browser/interact sessions is not allowed by default in your country. Please check your server configuration."+ : "Use of headers, actions, the agent, and browser/interact sessions is not allowed by default in your country. Please contact us at [email protected]",});}next();}+export function restrictedOptionsCountryCheck(+ req: RequestWithAuth<any, any, any>,+ res: Response,+ next: NextFunction,+) {+ const usesRestrictedOptions =+ req.body &&+ (req.body.actions ||+ (req.body.headers &&+ typeof req.body.headers === "object" &&+ Object.keys(req.body.headers).length > 0) ||+ req.body.agent ||+ req.body.jsonOptions?.agent ||+ req.body.extract?.agent ||+ req.body.scrapeOptions?.actions ||+ (req.body.scrapeOptions?.headers &&+ typeof req.body.scrapeOptions.headers === "object" &&+ Object.keys(req.body.scrapeOptions.headers).length > 0) ||+ req.body.scrapeOptions?.agent ||+ req.body.scrapeOptions?.jsonOptions?.agent ||+ req.body.scrapeOptions?.extract?.agent);++ if (!usesRestrictedOptions) {+ return next();+ }++ countryCheck(req, res, next);+}+export function isValidJobId(jobId: string | undefined): jobId is string {return typeof jobId === "string" && isUuid(jobId);}diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex bede8a778..32f7305f7 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -29,7 +29,7 @@ import {checkCreditsMiddleware,blocklistMiddleware,scrapeBlocklistMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,idempotencyMiddleware,requestTimingMiddleware,validateJobIdParam,@@ -52,7 +52,7 @@ v1Router.post("/scrape",authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),scrapeBlocklistMiddleware,wrap(scrapeController),@@ -62,7 +62,7 @@ v1Router.post("/crawl",authMiddleware(RateLimiterMode.Crawl),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(),scrapeBlocklistMiddleware,idempotencyMiddleware,@@ -73,7 +73,7 @@ v1Router.post("/batch/scrape",authMiddleware(RateLimiterMode.Scrape),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(),blocklistMiddleware,idempotencyMiddleware,@@ -84,7 +84,7 @@ v1Router.post("/search",authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),wrap(searchController),);@@ -157,7 +157,7 @@ v1Router.post(authMiddleware(RateLimiterMode.Extract),inferIntegrationMiddleware,deprecationMiddleware("v1_extract"),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(20),wrap(extractController),);@@ -173,7 +173,7 @@ v1Router.post("/llmstxt",authMiddleware(RateLimiterMode.Scrape),deprecationMiddleware("v1_llmstxt"),- countryCheck,+ restrictedOptionsCountryCheck,blocklistMiddleware,wrap(generateLLMsTextController),);@@ -189,7 +189,7 @@ v1Router.post("/deep-research",authMiddleware(RateLimiterMode.Crawl),deprecationMiddleware("v1_deep_research"),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),wrap(deepResearchController),);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex b0698fcd8..feeaebbeb 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -45,6 +45,7 @@ import {blocklistMiddleware,scrapeBlocklistMiddleware,countryCheck,+ restrictedOptionsCountryCheck,idempotencyMiddleware,requestTimingMiddleware,wrap,@@ -196,7 +197,7 @@ v2Router.post(allowAgentManagedKey: true,}),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),blocklistMiddleware,wrap(searchController),@@ -226,7 +227,7 @@ v2Router.get(v2Router.post("/parse/upload-url",authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),- countryCheck,+ restrictedOptionsCountryCheck,wrap(parseUploadUrlController),);@@ -240,7 +241,7 @@ v2Router.put(v2Router.post("/parse",authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),parsePayloadMiddleware,inferIntegrationMiddleware,@@ -254,7 +255,7 @@ v2Router.post(allowAgentManagedKey: true,}),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),scrapeBlocklistMiddleware,wrap(scrapeController),@@ -270,6 +271,7 @@ v2Router.get(v2Router.post("/scrape/:jobId/interact",authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),+ countryCheck,validateJobIdParam,inferIntegrationMiddleware,wrap(scrapeInteractController),@@ -286,7 +288,7 @@ v2Router.post("/batch/scrape",authMiddleware(RateLimiterMode.Scrape, { allowAgentManagedKey: true }),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(),blocklistMiddleware,wrap(batchScrapeController),@@ -305,7 +307,7 @@ v2Router.post("/crawl",authMiddleware(RateLimiterMode.Crawl),inferIntegrationMiddleware,- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(),scrapeBlocklistMiddleware,idempotencyMiddleware,@@ -393,7 +395,7 @@ v2Router.post(authMiddleware(RateLimiterMode.Extract),inferIntegrationMiddleware,deprecationMiddleware("v2_extract"),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(20),blocklistMiddleware,wrap(extractController),@@ -558,7 +560,7 @@ v2Router.post(v2Router.post("/monitor",authMiddleware(RateLimiterMode.Crawl),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),blocklistMiddleware,wrap(createMonitorController),@@ -587,7 +589,7 @@ v2Router.get(v2Router.patch("/monitor/:monitorId",authMiddleware(RateLimiterMode.Crawl),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),blocklistMiddleware,wrap(updateMonitorController),@@ -602,7 +604,7 @@ v2Router.delete(v2Router.post("/monitor/:monitorId/run",authMiddleware(RateLimiterMode.Crawl),- countryCheck,+ restrictedOptionsCountryCheck,checkCreditsMiddleware(1),blocklistMiddleware,wrap(runMonitorController),@@ -665,6 +667,7 @@ v2Router.get(v2Router.post(["/browser/:sessionId/execute", "/interact/:sessionId/execute"],authMiddleware(RateLimiterMode.BrowserExecute),+ countryCheck,wrap(browserExecuteController),);