Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add the usOnly team flag for US-only processing (#5545)
Gergő Móricz02a6f701baf3
diff --git a/apps/api/src/__tests__/snips/v2/us-only.test.ts b/apps/api/src/__tests__/snips/v2/us-only.test.tsnew file mode 100644index 000000000..21eb815a8--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/us-only.test.ts@@ -0,0 +1,79 @@+import { describeIf, TEST_PRODUCTION } from "../lib";+import { scrape, scrapeRaw, scrapeTimeout, idmux, Identity } from "./lib";++describeIf(TEST_PRODUCTION)("US-only teams", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({+ name: "us-only/scrape",+ concurrency: 100,+ credits: 1000000,+ flags: { usOnly: true },+ });+ }, 10000 + scrapeTimeout);++ it.concurrent(+ "scrapes markdown and skips the query format with a warning",+ async () => {+ const response = await scrape(+ {+ url: "https://firecrawl.dev",+ formats: [+ "markdown",+ { type: "query", prompt: "What is Firecrawl?" },+ ],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.answer).toBeUndefined();+ expect(response.warning).toContain(+ "Query mode is not supported for teams restricted to US-only processing.",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "skips the highlights format with a warning",+ async () => {+ const response = await scrape(+ {+ url: "https://firecrawl.dev",+ formats: [+ "markdown",+ { type: "highlights", query: "What is Firecrawl?" },+ ],+ },+ identity,+ );++ expect(response.markdown).toBeDefined();+ expect(response.highlights).toBeUndefined();+ expect(response.warning).toContain(+ "Query mode is not supported for teams restricted to US-only processing.",+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "refuses X/Twitter URLs",+ async () => {+ const response = await scrapeRaw(+ { url: "https://x.com/NASA", timeout: scrapeTimeout },+ identity,+ );++ expect(response.statusCode).toBe(403);+ expect(response.body.success).toBe(false);+ expect(response.body.code).toBe("US_ONLY_UNAVAILABLE");+ expect(response.body.error).toContain(+ "X/Twitter scraping is not available for teams restricted to US-only processing.",+ );+ },+ scrapeTimeout,+ );+});diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex f9fc32e2e..70489f63d 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -370,7 +370,10 @@ async function scrapeControllerInner(});}- if (e.code === "unsafe_domain_blocked") {+ if (+ e.code === "unsafe_domain_blocked" ||+ e.code === "US_ONLY_UNAVAILABLE"+ ) {return res.status(403).json({success: false,code: e.code,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 6d8a199ba..a71657a35 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1414,6 +1414,9 @@ export type TeamFlags = {// The org's contract bars training on its data: FirePDF never captures its// documents (buildFirePdfTrainingCapture).trainingRestricted?: boolean;+ // Every request is processed by US-based vendors only. A route with no US+ // vendor is refused or skipped with a warning.+ usOnly?: boolean;organizationDataSourceAccess?: Record<string,{diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex 6ea287874..839933c80 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -652,6 +652,17 @@ export async function parseController(});}+ if (e.code === "US_ONLY_UNAVAILABLE") {+ setSpanAttributes(span, {+ "parse.status_code": 403,+ });+ return res.status(403).json({+ success: false,+ code: e.code,+ error: e.message,+ });+ }+const statusCode = timeoutErr ? 408 : 500;setSpanAttributes(span, {"parse.status_code": statusCode,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 1f89cd99a..f12f2ae9b 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -541,7 +541,10 @@ export async function scrapeController(});}- if (e.code === "unsafe_domain_blocked") {+ if (+ e.code === "unsafe_domain_blocked" ||+ e.code === "US_ONLY_UNAVAILABLE"+ ) {setSpanAttributes(span, {"scrape.status_code": 403,});diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 270262f0f..859afcbd5 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2189,6 +2189,9 @@ export type TeamFlags = {// The org's contract bars training on its data: FirePDF never captures its// documents (buildFirePdfTrainingCapture).trainingRestricted?: boolean;+ // Every request is processed by US-based vendors only. A route with no US+ // vendor is refused or skipped with a warning.+ usOnly?: boolean;organizationDataSourceAccess?: Record<string,{diff --git a/apps/api/src/lib/error-serde.ts b/apps/api/src/lib/error-serde.tsindex 0853e630b..d76dea91e 100644--- a/apps/api/src/lib/error-serde.ts+++ b/apps/api/src/lib/error-serde.ts@@ -42,6 +42,7 @@ import {PromptInjectionDetectedError,JsonExtractionContentTooLargeError,XTwitterConfigurationError,+ UsOnlyUnavailableError,ExchangeRefusedError,} from "../scraper/scrapeURL/error";import { UnsafeDomainBlockedError } from "./threat-protection/error";@@ -96,6 +97,7 @@ const errorMap: Record<ErrorCodes, any> = {SCRAPE_PROMPT_INJECTION_DETECTED: PromptInjectionDetectedError,SCRAPE_JSON_CONTENT_TOO_LARGE: JsonExtractionContentTooLargeError,SCRAPE_X_TWITTER_CONFIGURATION_ERROR: XTwitterConfigurationError,+ US_ONLY_UNAVAILABLE: UsOnlyUnavailableError,MAP_FAILED: MapFailedError,CONCURRENCY_QUEUE_TIMEOUT: ConcurrencyQueueTimeoutError,unsafe_domain_blocked: UnsafeDomainBlockedError,diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.tsindex b647e18e7..fa4c991bc 100644--- a/apps/api/src/lib/error.ts+++ b/apps/api/src/lib/error.ts@@ -43,6 +43,7 @@ export type ErrorCodes =| "SCRAPE_PROMPT_INJECTION_DETECTED"| "SCRAPE_JSON_CONTENT_TOO_LARGE"| "SCRAPE_X_TWITTER_CONFIGURATION_ERROR"+ | "US_ONLY_UNAVAILABLE"| "PARSE_UNSUPPORTED_OPTIONS"| "CRAWL_DENIAL"| "UNSUPPORTED_SITE"diff --git a/apps/api/src/lib/us-only.ts b/apps/api/src/lib/us-only.tsnew file mode 100644index 000000000..564070d83--- /dev/null+++ b/apps/api/src/lib/us-only.ts@@ -0,0 +1,6 @@+import type { TeamFlags } from "../controllers/v2/types";++/** Whether the team must be served by US-based vendors only. Any value but `true` reads as unrestricted. */+export function isUsOnly(flags: TeamFlags | undefined): boolean {+ return flags?.usOnly === true;+}diff --git a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.tsindex 206b2d224..3a6236083 100644--- a/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/x-twitter/index.ts@@ -8,7 +8,12 @@ import {} from "../../../../lib/xai-x-search";import { Meta } from "../..";import { EngineScrapeResult } from "..";-import { EngineError, XTwitterConfigurationError } from "../../error";+import {+ EngineError,+ UsOnlyUnavailableError,+ XTwitterConfigurationError,+} from "../../error";+import { isUsOnly } from "../../../../lib/us-only";import { safeMarkdownToHtml } from "../pdf/markdownToHtml";import { calculateCost } from "../../transformers/llmExtract";import {@@ -249,6 +254,12 @@ export async function scrapeURLWithXTwitter();}+ if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {+ throw new UsOnlyUnavailableError(+ "X/Twitter scraping is not available for teams restricted to US-only processing.",+ );+ }+if (!config.XAI_API_KEY) {throw new XTwitterConfigurationError();}diff --git a/apps/api/src/scraper/scrapeURL/error.ts b/apps/api/src/scraper/scrapeURL/error.tsindex 76baf6acf..a76c17016 100644--- a/apps/api/src/scraper/scrapeURL/error.ts+++ b/apps/api/src/scraper/scrapeURL/error.ts@@ -31,6 +31,22 @@ export class XTwitterConfigurationError extends TransportableError {}}+/** A route the team's US-only restriction rules out, because no US-based vendor serves it. */+export class UsOnlyUnavailableError extends TransportableError {+ constructor(message: string) {+ super("US_ONLY_UNAVAILABLE", message);+ }++ static deserialize(+ _: ErrorCodes,+ data: ReturnType<typeof this.prototype.serialize>,+ ) {+ const x = new UsOnlyUnavailableError(data.message);+ x.stack = data.stack;+ return x;+ }+}+export class NoEnginesLeftError extends TransportableError {public fallbackList: Engine[];diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 699f4d3c0..87b641732 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -60,6 +60,7 @@ import {ScrapeRetryLimitError,BrandingNotSupportedError,XTwitterConfigurationError,+ UsOnlyUnavailableError,ExchangeRefusedError,} from "./error";import { ScrapeRetryTracker } from "./retryTracker";@@ -1070,6 +1071,7 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {error.error instanceof NoCachedDataError ||error.error instanceof AgentIndexOnlyError ||error.error instanceof XTwitterConfigurationError ||+ error.error instanceof UsOnlyUnavailableError ||error.error instanceof ExchangeRefusedError ||error.error instanceof ThirdPartyDataTermsRequiredError) {diff --git a/apps/api/src/scraper/scrapeURL/transformers/query.ts b/apps/api/src/scraper/scrapeURL/transformers/query.tsindex 2e579ef88..ac0ea9a5e 100644--- a/apps/api/src/scraper/scrapeURL/transformers/query.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/query.ts@@ -6,6 +6,7 @@ import { Meta } from "..";import { getModel } from "../../../lib/generic-ai";import { config } from "../../../config";import { hasFormatOfType } from "../../../lib/format-utils";+import { isUsOnly } from "../../../lib/us-only";import { calculateCost } from "./llmExtract";import { modelPrices } from "../../../lib/extract/usage/model-prices";import {@@ -395,6 +396,14 @@ export async function performQuery(return document;}+ if (isUsOnly(meta.internalOptions.teamFlags ?? undefined)) {+ addWarning(+ document,+ "Query mode is not supported for teams restricted to US-only processing.",+ );+ return document;+ }+if (document.markdown === undefined) {document.warning ="Query mode is not supported without markdown content." +