Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): enforce the country check on agent and browser/interact routes (#5204)

Gergő Móricz146d3bf3eeb5
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 34004dcf5..6fa3890ea 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -300,6 +300,11 @@ jobs:
run: pnpm exec vitest run src/controllers/v2/__tests__/agent-location.test.ts --reporter=default
working-directory: apps/api
+ - name: Run country check route regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=default
+ working-directory: apps/api
+
- name: Install test site dependencies
run: pnpm install --frozen-lockfile
working-directory: apps/test-site
diff --git a/apps/api/src/__tests__/routes/country-check.routes.test.ts b/apps/api/src/__tests__/routes/country-check.routes.test.ts
new file mode 100644
index 000000000..91171b342
--- /dev/null
+++ b/apps/api/src/__tests__/routes/country-check.routes.test.ts
@@ -0,0 +1,176 @@
+import express from "express";
+import request from "supertest";
+
+const mocks = vi.hoisted(() => ({
+ lookup: vi.fn(),
+ auth: { team_id: "team-1" } as { team_id: string },
+ acuc: undefined as { flags: Record<string, unknown> } | undefined,
+}));
+
+vi.mock("geoip-country", () => ({ lookup: mocks.lookup }));
+
+vi.mock("../../config", async importOriginal => {
+ const actual = await importOriginal<typeof import("../../config")>();
+ return {
+ ...actual,
+ config: {
+ ...actual.config,
+ USE_DB_AUTHENTICATION: true,
+ RESTRICTED_COUNTRIES: ["XX"],
+ },
+ };
+});
+
+// Real routers and country checks; auth is stubbed and the credit check ends
+// the chain with a 204, so a 204 means the request got past the country check.
+vi.mock("../../routes/shared", async importOriginal => {
+ const actual = await importOriginal<typeof import("../../routes/shared")>();
+ return {
+ ...actual,
+ authMiddleware: () => (req: any, _res: any, next: any) => {
+ req.auth = mocks.auth;
+ req.acuc = mocks.acuc;
+ next();
+ },
+ checkCreditsMiddleware: () => (_req: any, res: any) =>
+ res.status(204).end(),
+ };
+});
+
+import { config } from "../../config";
+import { v2Router } from "../../routes/v2";
+
+const app = express();
+app.use(express.json());
+app.use("/v2", v2Router);
+
+const JOB_ID = "11111111-1111-4111-8111-111111111111";
+const SESSION_ID = "22222222-2222-4222-8222-222222222222";
+
+const ALWAYS_CHECKED: Array<[string, Record<string, unknown>]> = [
+ ["/v2/agent", { prompt: "find the pricing page" }],
+ ["/v2/agent", { prompt: "and now the docs", threadId: JOB_ID }],
+ ["/v2/interact", {}],
+ ["/v2/browser", {}],
+ [`/v2/scrape/${JOB_ID}/interact`, { code: "console.log(1)" }],
+ [`/v2/interact/${SESSION_ID}/execute`, { code: "console.log(1)" }],
+ [`/v2/browser/${SESSION_ID}/execute`, { code: "console.log(1)" }],
+];
+
+function fromCountry(country: string) {
+ mocks.lookup.mockReturnValue({ country });
+}
+
+beforeEach(() => {
+ mocks.lookup.mockReset();
+ mocks.auth = { team_id: "team-1" };
+ mocks.acuc = { flags: {} };
+ config.USE_DB_AUTHENTICATION = true;
+});
+
+describe("country check on agent and browser routes", () => {
+ for (const [path, body] of ALWAYS_CHECKED) {
+ it(`rejects POST ${path} ${JSON.stringify(body)} from a restricted country`, async () => {
+ fromCountry("XX");
+
+ const res = await request(app).post(path).send(body);
+
+ expect(res.status).toBe(403);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain(
+ "the agent, and browser/interact sessions is not allowed by default in your country",
+ );
+ expect(mocks.lookup).toHaveBeenCalled();
+ });
+ }
+
+ it("lets agent and interact requests from other countries through", async () => {
+ fromCountry("YY");
+
+ expect(
+ (await request(app).post("/v2/agent").send({ prompt: "x" })).status,
+ ).toBe(204);
+ expect((await request(app).post("/v2/interact").send({})).status).toBe(204);
+ const interact = await request(app)
+ .post("/v2/scrape/not-a-uuid/interact")
+ .send({ code: "console.log(1)" });
+ expect(interact.status).toBe(400);
+ expect(interact.body.error).toContain("Invalid job ID format");
+ });
+
+ it("rejects keyless scrape interact without a team ACUC", async () => {
+ fromCountry("XX");
+ mocks.auth = { team_id: "preview_keyless_127.0.0.1" };
+ mocks.acuc = undefined;
+
+ const res = await request(app)
+ .post(`/v2/scrape/${JOB_ID}/interact`)
+ .send({ code: "console.log(1)" });
+
+ expect(res.status).toBe(403);
+ });
+
+ it("honors the team's skipCountryCheck flag", async () => {
+ fromCountry("XX");
+ mocks.acuc = { flags: { skipCountryCheck: true } };
+
+ expect(
+ (await request(app).post("/v2/agent").send({ prompt: "x" })).status,
+ ).toBe(204);
+ expect((await request(app).post("/v2/browser").send({})).status).toBe(204);
+ });
+
+ it("points self-hosted deployments at their server configuration", async () => {
+ fromCountry("XX");
+ config.USE_DB_AUTHENTICATION = false;
+
+ const res = await request(app).post("/v2/agent").send({ prompt: "x" });
+
+ expect(res.status).toBe(403);
+ expect(res.body.error).toContain("check your server configuration");
+ });
+
+ it("uses the hosted contact copy otherwise", async () => {
+ fromCountry("XX");
+
+ const res = await request(app).post("/v2/agent").send({ prompt: "x" });
+
+ expect(res.body.error).toContain("[email protected]");
+ });
+});
+
+describe("country check on scrape-style routes", () => {
+ it("lets a plain scrape from a restricted country through", async () => {
+ fromCountry("XX");
+
+ const res = await request(app)
+ .post("/v2/scrape")
+ .send({ url: "https://example.com" });
+
+ expect(res.status).toBe(204);
+ expect(mocks.lookup).not.toHaveBeenCalled();
+ });
+
+ it("rejects a scrape with headers from a restricted country", async () => {
+ fromCountry("XX");
+
+ const res = await request(app)
+ .post("/v2/scrape")
+ .send({ url: "https://example.com", headers: { "x-test": "1" } });
+
+ expect(res.status).toBe(403);
+ });
+
+ it("rejects a crawl with scrape actions from a restricted country", async () => {
+ fromCountry("XX");
+
+ const res = await request(app)
+ .post("/v2/crawl")
+ .send({
+ url: "https://example.com",
+ scrapeOptions: { actions: [{ type: "wait", milliseconds: 1 }] },
+ });
+
+ expect(res.status).toBe(403);
+ });
+});
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 21131e0fb..4f138b19d 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -487,28 +487,6 @@ export function countryCheck(
return next();
}
- const couldBeRestricted =
- req.body &&
- (req.body.actions ||
- (req.body.headers &&
- typeof req.body.headers === "object" &&
- Object.keys(req.body.headers).length > 0) ||
- req.body.agent ||
- req.body.jsonOptions?.agent ||
- req.body.extract?.agent ||
- req.body.scrapeOptions?.actions ||
- (req.body.scrapeOptions?.headers &&
- typeof req.body.scrapeOptions.headers === "object" &&
- Object.keys(req.body.scrapeOptions.headers).length > 0) ||
- req.body.scrapeOptions?.agent ||
- req.body.scrapeOptions?.jsonOptions?.agent ||
- req.body.scrapeOptions?.extract?.agent ||
- req.path.startsWith("/v2/agent"));
-
- if (!couldBeRestricted) {
- return next();
- }
-
if (!req.ip) {
logger.warn("IP address not found, unable to check country");
return next();
@@ -529,14 +507,43 @@ export function countryCheck(
return res.status(403).json({
success: false,
error: isSelfHosted()
- ? "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please check your server configuration."
- : "Use of headers, actions, and the FIRE-1 agent is not allowed by default in your country. Please contact us at [email protected]",
+ ? "Use of headers, actions, the agent, and browser/interact sessions is not allowed by default in your country. Please check your server configuration."
+ : "Use of headers, actions, the agent, and browser/interact sessions is not allowed by default in your country. Please contact us at [email protected]",
});
}
next();
}
+export function restrictedOptionsCountryCheck(
+ req: RequestWithAuth<any, any, any>,
+ res: Response,
+ next: NextFunction,
+) {
+ const usesRestrictedOptions =
+ req.body &&
+ (req.body.actions ||
+ (req.body.headers &&
+ typeof req.body.headers === "object" &&
+ Object.keys(req.body.headers).length > 0) ||
+ req.body.agent ||
+ req.body.jsonOptions?.agent ||
+ req.body.extract?.agent ||
+ req.body.scrapeOptions?.actions ||
+ (req.body.scrapeOptions?.headers &&
+ typeof req.body.scrapeOptions.headers === "object" &&
+ Object.keys(req.body.scrapeOptions.headers).length > 0) ||
+ req.body.scrapeOptions?.agent ||
+ req.body.scrapeOptions?.jsonOptions?.agent ||
+ req.body.scrapeOptions?.extract?.agent);
+
+ if (!usesRestrictedOptions) {
+ return next();
+ }
+
+ countryCheck(req, res, next);
+}
+
export function isValidJobId(jobId: string | undefined): jobId is string {
return typeof jobId === "string" && isUuid(jobId);
}
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index bede8a778..32f7305f7 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -29,7 +29,7 @@ import {
checkCreditsMiddleware,
blocklistMiddleware,
scrapeBlocklistMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
idempotencyMiddleware,
requestTimingMiddleware,
validateJobIdParam,
@@ -52,7 +52,7 @@ v1Router.post(
"/scrape",
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
scrapeBlocklistMiddleware,
wrap(scrapeController),
@@ -62,7 +62,7 @@ v1Router.post(
"/crawl",
authMiddleware(RateLimiterMode.Crawl),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(),
scrapeBlocklistMiddleware,
idempotencyMiddleware,
@@ -73,7 +73,7 @@ v1Router.post(
"/batch/scrape",
authMiddleware(RateLimiterMode.Scrape),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(),
blocklistMiddleware,
idempotencyMiddleware,
@@ -84,7 +84,7 @@ v1Router.post(
"/search",
authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
wrap(searchController),
);
@@ -157,7 +157,7 @@ v1Router.post(
authMiddleware(RateLimiterMode.Extract),
inferIntegrationMiddleware,
deprecationMiddleware("v1_extract"),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(20),
wrap(extractController),
);
@@ -173,7 +173,7 @@ v1Router.post(
"/llmstxt",
authMiddleware(RateLimiterMode.Scrape),
deprecationMiddleware("v1_llmstxt"),
- countryCheck,
+ restrictedOptionsCountryCheck,
blocklistMiddleware,
wrap(generateLLMsTextController),
);
@@ -189,7 +189,7 @@ v1Router.post(
"/deep-research",
authMiddleware(RateLimiterMode.Crawl),
deprecationMiddleware("v1_deep_research"),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
wrap(deepResearchController),
);
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index b0698fcd8..feeaebbeb 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -45,6 +45,7 @@ import {
blocklistMiddleware,
scrapeBlocklistMiddleware,
countryCheck,
+ restrictedOptionsCountryCheck,
idempotencyMiddleware,
requestTimingMiddleware,
wrap,
@@ -196,7 +197,7 @@ v2Router.post(
allowAgentManagedKey: true,
}),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
blocklistMiddleware,
wrap(searchController),
@@ -226,7 +227,7 @@ v2Router.get(
v2Router.post(
"/parse/upload-url",
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
- countryCheck,
+ restrictedOptionsCountryCheck,
wrap(parseUploadUrlController),
);
@@ -240,7 +241,7 @@ v2Router.put(
v2Router.post(
"/parse",
authMiddleware(RateLimiterMode.Scrape, { allowKeyless: true }),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
parsePayloadMiddleware,
inferIntegrationMiddleware,
@@ -254,7 +255,7 @@ v2Router.post(
allowAgentManagedKey: true,
}),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
scrapeBlocklistMiddleware,
wrap(scrapeController),
@@ -270,6 +271,7 @@ v2Router.get(
v2Router.post(
"/scrape/:jobId/interact",
authMiddleware(RateLimiterMode.BrowserExecute, { allowKeyless: true }),
+ countryCheck,
validateJobIdParam,
inferIntegrationMiddleware,
wrap(scrapeInteractController),
@@ -286,7 +288,7 @@ v2Router.post(
"/batch/scrape",
authMiddleware(RateLimiterMode.Scrape, { allowAgentManagedKey: true }),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(),
blocklistMiddleware,
wrap(batchScrapeController),
@@ -305,7 +307,7 @@ v2Router.post(
"/crawl",
authMiddleware(RateLimiterMode.Crawl),
inferIntegrationMiddleware,
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(),
scrapeBlocklistMiddleware,
idempotencyMiddleware,
@@ -393,7 +395,7 @@ v2Router.post(
authMiddleware(RateLimiterMode.Extract),
inferIntegrationMiddleware,
deprecationMiddleware("v2_extract"),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(20),
blocklistMiddleware,
wrap(extractController),
@@ -558,7 +560,7 @@ v2Router.post(
v2Router.post(
"/monitor",
authMiddleware(RateLimiterMode.Crawl),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
blocklistMiddleware,
wrap(createMonitorController),
@@ -587,7 +589,7 @@ v2Router.get(
v2Router.patch(
"/monitor/:monitorId",
authMiddleware(RateLimiterMode.Crawl),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
blocklistMiddleware,
wrap(updateMonitorController),
@@ -602,7 +604,7 @@ v2Router.delete(
v2Router.post(
"/monitor/:monitorId/run",
authMiddleware(RateLimiterMode.Crawl),
- countryCheck,
+ restrictedOptionsCountryCheck,
checkCreditsMiddleware(1),
blocklistMiddleware,
wrap(runMonitorController),
@@ -665,6 +667,7 @@ v2Router.get(
v2Router.post(
["/browser/:sessionId/execute", "/interact/:sessionId/execute"],
authMiddleware(RateLimiterMode.BrowserExecute),
+ countryCheck,
wrap(browserExecuteController),
);