Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add the usOnly team flag for US-only processing (#5545)

Gergő Móricz02a6f701baf3
diff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.ts
new file mode 100644
index 000000000..21eb815a8
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts
@@ -0,0 +1,79 @@
+import { describeIf, TEST_PRODUCTION } from "../lib";
+import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";
+
+describeIf(TEST_PRODUCTION)("US-only teams", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({
+ name: "us-only/scrape",
+ concurrency: 100,
+ credits: 1000000,
+ flags: { usOnly: true },
+ });
+ }, 10000 + scrapeTimeout);
+
+ it.concurrent(
+ "scrapes markdown and skips the query format with a warning",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: [
+ "markdown",
+ { type: "query", prompt: "What is Firecrawl?" },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.answer).toBeUndefined();
+ expect(response.warning).toContain(
+ "Query mode is not supported for teams restricted to US-only processing.",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "skips the highlights format with a warning",
+ async () => {
+ const response = await scrape(
+ {
+ url: "https://firecrawl.dev",
+ formats: [
+ "markdown",
+ { type: "highlights", query: "What is Firecrawl?" },
+ ],
+ },
+ identity,
+ );
+
+ expect(response.markdown).toBeDefined();
+ expect(response.highlights).toBeUndefined();
+ expect(response.warning).toContain(
+ "Query mode is not supported for teams restricted to US-only processing.",
+ );
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "refuses X/Twitter URLs",
+ async () => {
+ const response = await scrapeRaw(
+ { url: "https://x.com/NASA", timeout: scrapeTimeout },
+ identity,
+ );
+
+ expect(response.statusCode).toBe(403);
+ expect(response.body.success).toBe(false);
+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");
+ expect(response.body.error).toContain(
+ "X/Twitter scraping is not available for teams restricted to US-only processing.",
+ );
+ },
+ scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index f9fc32e2e..70489f63d 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -370,7 +370,10 @@ async function scrapeControllerInner(
});
}
- if (e.code === "unsafe_domain_blocked") {
+ if (
+ e.code === "unsafe_domain_blocked" ||
+ e.code === "US_ONLY_UNAVAILABLE"
+ ) {
return res.status(403).json({
success: false,
code: e.code,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 6d8a199ba..a71657a35 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1414,6 +1414,9 @@ export type TeamFlags = {
// The org's contract bars training on its data: FirePDF never captures its
// documents (buildFirePdfTrainingCapture).
trainingRestricted?: boolean;
+ // Every request is processed by US-based vendors only. A route with no US
+ // vendor is refused or skipped with a warning.
+ usOnly?: boolean;
organizationDataSourceAccess?: Record<
string,
{
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index 6ea287874..839933c80 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -652,6 +652,17 @@ export async function parseController(
});
}
+ if (e.code === "US_ONLY_UNAVAILABLE") {
+ setSpanAttributes(span, {
+ "parse.status_code": 403,
+ });
+ return res.status(403).json({
+ success: false,
+ code: e.code,
+ error: e.message,
+ });
+ }
+
const statusCode = timeoutErr ? 408 : 500;
setSpanAttributes(span, {
"parse.status_code": statusCode,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 1f89cd99a..f12f2ae9b 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -541,7 +541,10 @@ export async function scrapeController(
});
}
- if (e.code === "unsafe_domain_blocked") {
+ if (
+ e.code === "unsafe_domain_blocked" ||
+ e.code === "US_ONLY_UNAVAILABLE"
+ ) {
setSpanAttributes(span, {
"scrape.status_code": 403,
});
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 270262f0f..859afcbd5 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2189,6 +2189,9 @@ export type TeamFlags = {
// The org's contract bars training on its data: FirePDF never captures its
// documents (buildFirePdfTrainingCapture).
trainingRestricted?: boolean;
+ // Every request is processed by US-based vendors only. A route with no US
+ // vendor is refused or skipped with a warning.
+ usOnly?: boolean;
organizationDataSourceAccess?: Record<
string,
{
diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.ts
index 0853e630b..d76dea91e 100644
--- a/apps/api/src/lib/error-serde.ts
+++ b/apps/api/src/lib/error-serde.ts
@@ -42,6 +42,7 @@ import {
PromptInjectionDetectedError,
JsonExtractionContentTooLargeError,
XTwitterConfigurationError,
+ UsOnlyUnavailableError,
ExchangeRefusedError,
} from "../scraper/scrapeURL/error";
import { UnsafeDomainBlockedError } from "./threat-protection/error";
@@ -96,6 +97,7 @@ const errorMap: Record<ErrorCodes, any> = {
SCRAPE_PROMPT_INJECTION_DETECTED: PromptInjectionDetectedError,
SCRAPE_JSON_CONTENT_TOO_LARGE: JsonExtractionContentTooLargeError,
SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,
+ US_ONLY_UNAVAILABLE: UsOnlyUnavailableError,
MAP_FAILED: MapFailedError,
CONCURRENCY_QUEUE_TIMEOUT: ConcurrencyQueueTimeoutError,
unsafe_domain_blocked: UnsafeDomainBlockedError,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index b647e18e7..fa4c991bc 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -43,6 +43,7 @@ export type ErrorCodes =
| "SCRAPE_PROMPT_INJECTION_DETECTED"
| "SCRAPE_JSON_CONTENT_TOO_LARGE"
| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"
+ | "US_ONLY_UNAVAILABLE"
| "PARSE_UNSUPPORTED_OPTIONS"
| "CRAWL_DENIAL"
| "UNSUPPORTED_SITE"
diff --git a/apps/api/src/lib/us-only.ts b/apps/api/src/lib/us-only.ts
new file mode 100644
index 000000000..564070d83
--- /dev/null
+++ b/apps/api/src/lib/us-only.ts
@@ -0,0 +1,6 @@
+import type { TeamFlags } from "../controllers/v2/types";
+
+/** Whether the team must be served by US-based vendors only. Any value but `true` reads as unrestricted. */
+export function isUsOnly(flags: TeamFlags | undefined): boolean {
+ return flags?.usOnly === true;
+}
diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
index 206b2d224..3a6236083 100644
--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts
@@ -8,7 +8,12 @@ import {
} from "../../../../lib/xai-x-search";
import { Meta } from "../..";
import { EngineScrapeResult } from "..";
-import { EngineError, XTwitterConfigurationError } from "../../error";
+import {
+ EngineError,
+ UsOnlyUnavailableError,
+ XTwitterConfigurationError,
+} from "../../error";
+import { isUsOnly } from "../../../../lib/us-only";
import { safeMarkdownToHtml } from "../pdf/markdownToHtml";
import { calculateCost } from "../../transformers/llmExtract";
import {
@@ -249,6 +254,12 @@ export async function scrapeURLWithXTwitter(
);
}
+ if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
+ throw new UsOnlyUnavailableError(
+ "X/Twitter scraping is not available for teams restricted to US-only processing.",
+ );
+ }
+
if (!config.XAI_API_KEY) {
throw new XTwitterConfigurationError();
}
diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.ts
index 76baf6acf..a76c17016 100644
--- a/apps/api/src/scraper/scrapeURL/error.ts
+++ b/apps/api/src/scraper/scrapeURL/error.ts
@@ -31,6 +31,22 @@ export class XTwitterConfigurationError extends TransportableError {
}
}
+/** A route the team's US-only restriction rules out, because no US-based vendor serves it. */
+export class UsOnlyUnavailableError extends TransportableError {
+ constructor(message: string) {
+ super("US_ONLY_UNAVAILABLE", message);
+ }
+
+ static deserialize(
+ _: ErrorCodes,
+ data: ReturnType<typeof this.prototype.serialize>,
+ ) {
+ const x = new UsOnlyUnavailableError(data.message);
+ x.stack = data.stack;
+ return x;
+ }
+}
+
export class NoEnginesLeftError extends TransportableError {
public fallbackList: Engine[];
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 699f4d3c0..87b641732 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -60,6 +60,7 @@ import {
ScrapeRetryLimitError,
BrandingNotSupportedError,
XTwitterConfigurationError,
+ UsOnlyUnavailableError,
ExchangeRefusedError,
} from "./error";
import { ScrapeRetryTracker } from "./retryTracker";
@@ -1070,6 +1071,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
error.error instanceof NoCachedDataError ||
error.error instanceof AgentIndexOnlyError ||
error.error instanceof XTwitterConfigurationError ||
+ error.error instanceof UsOnlyUnavailableError ||
error.error instanceof ExchangeRefusedError ||
error.error instanceof ThirdPartyDataTermsRequiredError
) {
diff --git a/apps/api/src/scraper/scrapeURL/transformers/query.ts b/apps/api/src/scraper/scrapeURL/transformers/query.ts
index 2e579ef88..ac0ea9a5e 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/query.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/query.ts
@@ -6,6 +6,7 @@ import { Meta } from "..";
import { getModel } from "../../../lib/generic-ai";
import { config } from "../../../config";
import { hasFormatOfType } from "../../../lib/format-utils";
+import { isUsOnly } from "../../../lib/us-only";
import { calculateCost } from "./llmExtract";
import { modelPrices } from "../../../lib/extract/usage/model-prices";
import {
@@ -395,6 +396,14 @@ export async function performQuery(
return document;
}
+ if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {
+ addWarning(
+ document,
+ "Query mode is not supported for teams restricted to US-only processing.",
+ );
+ return document;
+ }
+
if (document.markdown === undefined) {
document.warning =
"Query mode is not supported without markdown content." +