Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Honor Exchange terms overrides while preserving universal scrape gates (#5026)
Developers Digestf0dd0592a565
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 3cf971a23..c89a157ba 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -218,6 +218,17 @@ jobs:env:npm_config_ignore_scripts: 'true'+ - name: Run Exchange terms compatibility regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/services/alexandria/access.test.ts+ src/services/alexandria/terms.test.ts+ src/lib/exchange.test.ts+ src/__tests__/routes/exchange-agent-managed-key.routes.test.ts+ --maxWorkers=1 --reporter=default+ working-directory: apps/api+- name: Run monitor finalization regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: >-diff --git a/apps/api/src/lib/exchange.test.ts b/apps/api/src/lib/exchange.test.tsindex d2240c15f..b479c0020 100644--- a/apps/api/src/lib/exchange.test.ts+++ b/apps/api/src/lib/exchange.test.ts@@ -737,6 +737,7 @@ describe("Exchange terms acceptance", () => {{provider: "acme",required: true,+ exchangeRequired: false,terms: { ...ACME_TERMS, digest: DIGEST },},],@@ -773,6 +774,16 @@ describe("Exchange terms acceptance", () => {clearExchangeProvidersForTest();});+ it("still gates universal scrape when Exchange acceptance is optional", async () => {+ ledgerAnswers([]);+ expect(+ await getExchangeAccessForRequest({+ ...PROFILE_REQUEST,+ flags: { professionalProfileCompanyDataBeta: true },+ }),+ ).toMatchObject({ allowed: false, termsRequired: true, terms: ACME_TERMS });+ });+it("routes when the terms were accepted on the Exchange ledger", async () => {ledgerAnswers([{ version: ACME_TERMS.version, textHash: DIGEST }]);diff --git a/apps/api/src/services/alexandria/access.test.ts b/apps/api/src/services/alexandria/access.test.tsindex fc80fbe59..b1bfb20b0 100644--- a/apps/api/src/services/alexandria/access.test.ts+++ b/apps/api/src/services/alexandria/access.test.ts@@ -59,43 +59,49 @@ it("refuses a disabled provider and fails closed when agreements are unavailableexpect((await authorizeProviders("team", calls, {}))?.status).toBe(503);});-it("allows a current agreement accepted after owner revocation, then blocks a later revocation", async () => {- const terms = { ...requirement(true) };- Object.assign(terms.body.providers[0].terms, { digest: "a".repeat(64) });- const access = {- status: "disabled",- disabledReason: "revoked_by_organization_admin",- disabledAt: "2026-09-20T12:00:00Z",- termsKey: "fred",- termsVersion: "2026-01",- };- const flags = { organizationDataSourceAccess: { fred: access } };- let acceptance = {- provider: "fred",- revoked: false,- version: "2026-01",- textHash: "a".repeat(64),- acceptedAt: "2026-09-20T11:00:00Z",- };- mocks.request.mockImplementation(async ({ path }) =>- path.includes("requirements")- ? terms- : { status: 200, body: { providers: [acceptance] } },- );- expect(- (await authorizeProviders("team", calls, flags, "org"))?.body,- ).toMatchObject({ code: "THIRD_PARTY_DATA_TERMS_REQUIRED" });- acceptance = {- ...acceptance,- revoked: false,- acceptedAt: "2026-09-20T12:01:00Z",- };- expect(await authorizeProviders("team", calls, flags, "org")).toBeUndefined();- access.disabledAt = "2026-09-20T12:02:00Z";- expect((await authorizeProviders("team", calls, flags, "org"))?.status).toBe(- 403,- );-});+it.each([undefined, false])(+ "allows post-revocation acceptance with Exchange override %s, then blocks a later revocation",+ async exchangeRequired => {+ const terms = { ...requirement(true) };+ Object.assign(terms.body.providers[0], { exchangeRequired });+ Object.assign(terms.body.providers[0].terms, { digest: "a".repeat(64) });+ const access = {+ status: "disabled",+ disabledReason: "revoked_by_organization_admin",+ disabledAt: "2026-09-20T12:00:00Z",+ termsKey: "fred",+ termsVersion: "2026-01",+ };+ const flags = { organizationDataSourceAccess: { fred: access } };+ let acceptance = {+ provider: "fred",+ revoked: false,+ version: "2026-01",+ textHash: "a".repeat(64),+ acceptedAt: "2026-09-20T11:00:00Z",+ };+ mocks.request.mockImplementation(async ({ path }) =>+ path.includes("requirements")+ ? terms+ : { status: 200, body: { providers: [acceptance] } },+ );+ expect(+ (await authorizeProviders("team", calls, flags, "org"))?.body,+ ).toMatchObject({ code: "THIRD_PARTY_DATA_TERMS_REQUIRED" });+ acceptance = {+ ...acceptance,+ revoked: false,+ acceptedAt: "2026-09-20T12:01:00Z",+ };+ expect(+ await authorizeProviders("team", calls, flags, "org"),+ ).toBeUndefined();+ access.disabledAt = "2026-09-20T12:02:00Z";+ expect(+ (await authorizeProviders("team", calls, flags, "org"))?.status,+ ).toBe(403);+ },+);it.each([[@@ -420,3 +426,47 @@ describe("paid-plan-only capabilities", () => {);});});++it("allows optional Exchange terms without looking up acceptance", async () => {+ const response = requirement(true);+ Object.assign(response.body.providers[0], { exchangeRequired: false });+ mocks.request.mockResolvedValue(response);+ expect(await authorizeProviders("team", calls, {}, "org")).toBeUndefined();+ expect(mocks.request).toHaveBeenCalledTimes(1);+});++it.each(["disabled", "suspended"])(+ "still refuses %s access when Exchange terms are optional",+ async status => {+ const response = requirement(true);+ Object.assign(response.body.providers[0], { exchangeRequired: false });+ mocks.request.mockResolvedValue(response);+ expect(+ (+ await authorizeProviders(+ "team",+ calls,+ {+ organizationDataSourceAccess: {+ fred: {+ status,+ disabledReason: "disabled_by_organization_admin",+ },+ },+ },+ "org",+ )+ )?.status,+ ).toBe(403);+ expect(mocks.request).toHaveBeenCalledTimes(1);+ },+);++it("honors a required Exchange override", async () => {+ const response = requirement(false);+ Object.assign(response.body.providers[0], { exchangeRequired: true });+ mocks.request.mockResolvedValue(response);+ expect((await authorizeProviders("team", calls, {}))?.body).toMatchObject({+ code: "THIRD_PARTY_DATA_TERMS_REQUIRED",+ });+});diff --git a/apps/api/src/services/alexandria/access.ts b/apps/api/src/services/alexandria/access.tsindex 5491e4ba2..65135f7e3 100644--- a/apps/api/src/services/alexandria/access.ts+++ b/apps/api/src/services/alexandria/access.ts@@ -17,6 +17,7 @@ const requirementsSchema = z.object({z.object({provider: z.string(),required: z.boolean(),+ exchangeRequired: z.boolean().optional(),terms: z.object({key: z.string(),@@ -70,12 +71,18 @@ export async function authorizeProviders(let ledger: Map<string, LedgerAcceptance> | undefined;for (const item of parsed.data.providers) {+ const required = item.exchangeRequired ?? item.required;const access = flags?.organizationDataSourceAccess?.[item.provider];if (access && access.status !== "enabled") {const revokedByOwner =access.status === "disabled" &&access.disabledReason === "revoked_by_organization_admin";- if (revokedByOwner && item.required && item.terms && orgId !== null) {+ if (+ revokedByOwner &&+ (item.required || required) &&+ item.terms &&+ orgId !== null+ ) {ledger ??= await acceptedProviders(teamId, orgId);const accepted = ledger.get(item.provider);if (@@ -93,7 +100,7 @@ export async function authorizeProviders(`Access to ${item.provider} is disabled for this organization.`,);}- if (!item.required || !item.terms) continue;+ if (!required || !item.terms) continue;if (access?.termsKey === item.terms.key &&access?.termsVersion === item.terms.versiondiff --git a/apps/api/src/services/alexandria/terms.test.ts b/apps/api/src/services/alexandria/terms.test.tsnew file mode 100644index 000000000..bdc6ea63b--- /dev/null+++ b/apps/api/src/services/alexandria/terms.test.ts@@ -0,0 +1,76 @@+const mocks = vi.hoisted(() => ({ request: vi.fn() }));+vi.mock("./client", () => ({ exchangeRequest: mocks.request }));+import { acceptProviderTerms } from "./terms";++const terms = { key: "shopify", version: "2026-01", digest: "a".repeat(64) };+const input = {+ teamId: "team",+ orgId: "org",+ apiKeyId: "key",+ body: {+ provider: "shopify",+ version: terms.version,+ digest: terms.digest,+ confirmed: true as const,+ },+};++beforeEach(() => vi.resetAllMocks());++it.each([+ [true, false],+ [false, true],+])(+ "records explicit acceptance with universal requirement %s and Exchange override %s",+ async (required, exchangeRequired) => {+ mocks.request+ .mockResolvedValueOnce({+ status: 200,+ body: {+ providers: [+ {+ provider: "shopify",+ required,+ exchangeRequired,+ terms,+ },+ ],+ },+ })+ .mockResolvedValueOnce({+ status: 201,+ body: { id: "event", occurred_at: "2026-10-08T12:00:00Z" },+ });+ expect((await acceptProviderTerms(input)).status).toBe(200);+ expect(mocks.request).toHaveBeenLastCalledWith(+ expect.objectContaining({+ path: "/v1/provider-terms/events",+ body: expect.objectContaining({+ event_type: "accepted",+ version: terms.version,+ text_hash: terms.digest,+ }),+ }),+ );+ },+);++it("still rejects stale explicit acceptance when Exchange terms are optional", async () => {+ mocks.request.mockResolvedValueOnce({+ status: 200,+ body: {+ providers: [+ { provider: "shopify", required: true, exchangeRequired: false, terms },+ ],+ },+ });+ expect(+ (+ await acceptProviderTerms({+ ...input,+ body: { ...input.body, version: "old" },+ })+ ).status,+ ).toBe(409);+ expect(mocks.request).toHaveBeenCalledTimes(1);+});diff --git a/apps/api/src/services/alexandria/terms.ts b/apps/api/src/services/alexandria/terms.tsindex 8f042bb7c..4c57e1d5b 100644--- a/apps/api/src/services/alexandria/terms.ts+++ b/apps/api/src/services/alexandria/terms.ts@@ -24,6 +24,7 @@ const requirementsSchema = z.object({z.object({provider: z.string(),required: z.boolean(),+ exchangeRequired: z.boolean().optional(),terms: z.object({key: z.string(),@@ -184,7 +185,7 @@ export async function acceptProviderTerms(input: {503,"Provider agreements are unavailable. Nothing was accepted.",);- if (!item.required || !item.terms)+ if (!(item.required || item.exchangeRequired) || !item.terms)return refusal(400, `${provider} requires no agreement.`);if (item.terms.version !== version || item.terms.digest !== digest)return refusal(