Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): thread org context through the scrape pipeline (#4064)
Rafael Miller93387da4fbdf
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex d050b99d3..9ce2cb5ab 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -77,6 +77,7 @@ const mockPreviewACUC: (api_key: "preview",api_key_id: 0,team_id,+ org_id: "preview",rate_limits: {crawl: 2,scrape: 10,@@ -102,6 +103,7 @@ const mockACUC: () => AuthCreditUsageChunk = () => ({api_key: "bypass",api_key_id: 0,team_id: "bypass",+ org_id: "bypass",rate_limits: {crawl: 99999999,scrape: 99999999,diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.tsindex a1a9c4682..75ba03b18 100644--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts@@ -61,6 +61,7 @@ export async function crawlMonitorController(internalOptions: {disableSmartWaitCache: true,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention: false,},diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex ee19ad91b..3cc6ec708 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -180,6 +180,7 @@ export async function crawlController(req: Request, res: Response) {if (isUrlBlocked(url, auth.chunk?.flags ?? null, {team_id: auth.chunk?.team_id ?? team_id,+ org_id: auth.chunk?.org_id ?? null,origin: req.body?.origin ?? null,})) {@@ -224,6 +225,7 @@ export async function crawlController(req: Request, res: Response) {team_id,);internalOptions.disableSmartWaitCache = true; // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter+ internalOptions.orgId = auth.chunk?.org_id ?? null;internalOptions.saveScrapeResultToGCS = process.env.GCS_FIRE_ENGINE_BUCKET_NAME? truediff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 4679ed2a0..8ae9a1a7c 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -41,6 +41,7 @@ async function scrapeHelper(extractorOptions: ExtractorOptions,timeout: number,flags: TeamFlags,+ org_id: string | null,apiKeyId: number | null,): Promise<{success: boolean;@@ -70,6 +71,7 @@ async function scrapeHelper(if (isUrlBlocked(url, flags, {team_id,+ org_id,origin: req.body?.origin ?? null,})) {@@ -88,6 +90,7 @@ async function scrapeHelper(team_id,);+ internalOptions.orgId = org_id;internalOptions.saveScrapeResultToGCS = process.env.GCS_FIRE_ENGINE_BUCKET_NAME? true@@ -299,6 +302,7 @@ export async function scrapeController(req: Request, res: Response) {extractorOptions,timeout,chunk?.flags ?? null,+ chunk?.org_id ?? null,chunk?.api_key_id ?? null,);diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 19463249b..55bd5ee8b 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -37,6 +37,7 @@ async function searchHelper(pageOptions: PageOptions,searchOptions: SearchOptions,flags: TeamFlags,+ org_id: string | null,api_key_id: number | null,): Promise<{success: boolean;@@ -81,6 +82,7 @@ async function searchHelper(crawlerOptions,team_id,);+ internalOptions.orgId = org_id;if (justSearch) {const searchCredits = Math.ceil(res.length / 10) * 2;@@ -103,6 +105,7 @@ async function searchHelper(r =>!isUrlBlocked(r.url, flags, {team_id,+ org_id,origin: req.body?.origin ?? null,}),);@@ -269,6 +272,7 @@ export async function searchController(req: Request, res: Response) {pageOptions,searchOptions,chunk?.flags ?? null,+ chunk?.org_id ?? null,chunk?.api_key_id ?? null,);const endTime = new Date().getTime();diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 8e5d4750a..d20906b9f 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -121,6 +121,7 @@ export async function batchScrapeController(if (!isUrlBlocked(nu, req.acuc?.flags ?? null, {team_id: req.auth.team_id,+ org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,})) {@@ -138,6 +139,7 @@ export async function batchScrapeController(req.body.urls?.some((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {team_id: req.auth.team_id,+ org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,}),)@@ -250,6 +252,7 @@ export async function batchScrapeController(...internalOptions,disableSmartWaitCache: true,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex fd730e57f..877cc43f8 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -201,6 +201,7 @@ export async function crawlController(...internalOptions,disableSmartWaitCache: true,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex 75b72462b..b1b778a31 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -124,6 +124,7 @@ export async function extractController(req.body.urls?.filter((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {team_id: req.auth.team_id,+ org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,}),) ?? [];diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 052d88f0b..b9ec77ecd 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -93,6 +93,7 @@ export async function getMapResults({includeSubdomains = true,crawlerOptions = {},teamId,+ orgId,origin,includeMetadata = false,allowExternalLinks,@@ -114,6 +115,7 @@ export async function getMapResults({includeSubdomains?: boolean;crawlerOptions?: any;teamId: string;+ orgId?: string | null;origin?: string;includeMetadata?: boolean;allowExternalLinks?: boolean;@@ -146,7 +148,7 @@ export async function getMapResults({...(location ? { location } : {}),...(headers ? { headers } : {}),}),- internalOptions: { teamId },+ internalOptions: { teamId, orgId: orgId ?? null },team_id: teamId,createdAt: Date.now(),};@@ -445,6 +447,7 @@ export async function mapController(crawlerOptions: req.body,origin: req.body.origin,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,abort: abort.signal,mock: req.body.useMock,filterByPath: req.body.filterByPath !== false,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 512d8280b..d5bc60f09 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -227,6 +227,7 @@ export async function scrapeController(bypassBilling: isDirectToBullMQ,zeroDataRetention,teamFlags: req.acuc?.flags ?? null,+ orgId: req.acuc?.org_id ?? null,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,},diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 8656a4371..129b83e48 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -47,6 +47,7 @@ export async function searchAndScrapeSearchResult(query: string,options: {teamId: string;+ orgId?: string | null;origin: string;timeout: number;scrapeOptions: any;@@ -77,6 +78,7 @@ export async function searchAndScrapeSearchResult(})),{teamId: options.teamId,+ orgId: options.orgId ?? null,origin: options.origin,timeout: options.timeout,scrapeOptions,@@ -235,6 +237,7 @@ export async function searchController(},{teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,origin: req.body.origin,integration: req.body.integration,apiKeyId: req.acuc?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2284d4d45..4d9c63011 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1256,7 +1256,7 @@ export type AuthCreditUsageChunk = {api_key: string;api_key_id: number;team_id: string;- org_id?: string | null;+ org_id: string;plan_priority: {bucketLimit: number;planModifier: number;@@ -1452,6 +1452,7 @@ function fromLegacyCrawlerOptions(internalOptions: {v0CrawlOnlyUrls: x.returnOnlyUrls,teamId,+ orgId: null,},};}@@ -1517,6 +1518,7 @@ export function fromLegacyScrapeOptions(atsv: pageOptions.atsv,v0DisableJsDom: pageOptions.disableJsDom,teamId,+ orgId: null,},// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks};diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 7228d6921..d393df0e5 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -142,6 +142,7 @@ export async function batchScrapeController(if (!isUrlBlocked(nu, req.acuc?.flags ?? null, {team_id: req.auth.team_id,+ org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,})) {@@ -159,6 +160,7 @@ export async function batchScrapeController(req.body.urls?.some((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {team_id: req.auth.team_id,+ org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,}),)@@ -271,6 +273,7 @@ export async function batchScrapeController(internalOptions: {disableSmartWaitCache: true,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.tsindex f3cfe561a..a304860ac 100644--- a/apps/api/src/controllers/v2/crawl-cancel.ts+++ b/apps/api/src/controllers/v2/crawl-cancel.ts@@ -41,6 +41,7 @@ export async function crawlCancelController(scrapeOptions: scrapeOptions.parse({}),internalOptions: {teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,},};diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex 312a3af9a..fb7539f97 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -67,6 +67,7 @@ export async function crawlParamsPreviewController(basePrompt: parsedBody.prompt,url: parsedBody.url,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,logger,limit: 50,diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 5fc1761ce..a0f1f9b1e 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -170,6 +170,7 @@ export async function crawlController(basePrompt: req.body.prompt,url: req.body.url,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,flags: req.acuc?.flags ?? null,logger,limit: 50,@@ -262,6 +263,7 @@ export async function crawlController(internalOptions: {disableSmartWaitCache: true,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,zeroDataRetention,agentIndexOnly: (req as any).agentIndexOnly ?? false,diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex 59476f464..fb21af95f 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -67,6 +67,7 @@ export async function extractController(req.body.urls?.filter((url: string) =>isUrlBlocked(url, req.acuc?.flags ?? null, {team_id: req.auth.team_id,+ org_id: req.acuc?.org_id ?? null,origin: req.body.origin ?? null,}),) ?? [];diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex b85681fb2..eb02d2ae9 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -167,6 +167,7 @@ export async function mapController(},origin: req.body.origin,teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,allowExternalLinks: req.body.allowExternalLinks,abort: abort.signal,mock: req.body.useMock,diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.tsindex f47a15917..9e879c2c3 100644--- a/apps/api/src/controllers/v2/parse.ts+++ b/apps/api/src/controllers/v2/parse.ts@@ -512,6 +512,7 @@ export async function parseController(bypassBilling: isDirectToBullMQ || !shouldBill,zeroDataRetention,teamFlags: req.acuc?.flags ?? null,+ orgId: req.acuc?.org_id ?? null,teamConcurrency: req.acuc?.concurrency ?? null,uploadedFile: file,forceEngine,diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 57b22730b..3ce6e1f77 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -328,6 +328,7 @@ export async function scrapeController(bypassBilling: isDirectToBullMQ || !shouldBill,zeroDataRetention,teamFlags: req.acuc?.flags ?? null,+ orgId: req.acuc?.org_id ?? null,teamConcurrency: req.acuc?.concurrency ?? null,agentIndexOnly: (req as any).agentIndexOnly ?? false,threatProtection: threatProtection.policy ?? undefined,diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex f52f45fd7..55dba5ad6 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -218,6 +218,7 @@ export async function searchController(},{teamId: req.auth.team_id,+ orgId: req.acuc?.org_id ?? null,origin: req.body.origin,integration: req.body.integration,apiKeyId: req.acuc?.api_key_id ?? null,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f5e15000f..e73875a29 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1663,6 +1663,7 @@ function fromV0CrawlerOptions(internalOptions: {v0CrawlOnlyUrls: x.returnOnlyUrls,teamId,+ orgId: null,},};}@@ -1727,6 +1728,7 @@ export function fromV0ScrapeOptions(atsv: pageOptions.atsv,v0DisableJsDom: pageOptions.disableJsDom,teamId,+ orgId: null,...(extractorOptions !== undefined &&extractorOptions.mode.includes("llm-extraction")? {@@ -1839,6 +1841,7 @@ export function fromV1ScrapeOptions(}),internalOptions: {teamId,+ orgId: null,v1Agent: v1ScrapeOptions.agent,v1JSONSystemPrompt: (v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extractdiff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.tsindex 9af4190d5..e86c8ba23 100644--- a/apps/api/src/db/schema/public.ts+++ b/apps/api/src/db/schema/public.ts@@ -96,6 +96,7 @@ export const batch_scrapes = pgTable("batch_scrapes", {export const blocklist = pgTable("blocklist", {id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),data: jsonb("data").notNull(),+ org_id: uuid("org_id"),});export const blocklist_hits = pgTable("blocklist_hits", {diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.tsindex 16b36b300..ef0853e44 100644--- a/apps/api/src/lib/deep-research/deep-research-service.ts+++ b/apps/api/src/lib/deep-research/deep-research-service.ts@@ -133,6 +133,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {searchQuery.query,{teamId: options.teamId,+ orgId: acuc?.org_id ?? null,origin: "deep-research",timeout: 10000,scrapeOptions: {diff --git a/apps/api/src/lib/engpicker.ts b/apps/api/src/lib/engpicker.tsindex 74fddfdb8..1e19e08c5 100644--- a/apps/api/src/lib/engpicker.ts+++ b/apps/api/src/lib/engpicker.ts@@ -48,6 +48,7 @@ async function evaluateURL({forceEngine: engine,teamId: "engpicker",+ orgId: null,},new CostTracking(),);diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 1da8e2740..0246ab098 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -17,6 +17,7 @@ import { scrapeQueue } from "../../services/worker/nuq-router";interface ScrapeDocumentOptions {url: string;teamId: string;+ orgId?: string | null;origin: string;timeout: number;isSingleUrl?: boolean;@@ -40,6 +41,7 @@ export async function scrapeDocument(if (isUrlBlocked(options.url, options.flags ?? null, {team_id: options.teamId,+ org_id: options.orgId ?? null,origin: options.origin,})) {@@ -64,6 +66,7 @@ export async function scrapeDocument(}),internalOptions: {teamId: options.teamId,+ orgId: options.orgId ?? null,saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME? true: false,diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex 08a0b0b77..7a4522099 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -17,6 +17,7 @@ import type { ThreatProtectionPolicy } from "../../threat-protection/types";interface ScrapeDocumentOptions {url: string;teamId: string;+ orgId?: string | null;origin: string;timeout: number;isSingleUrl?: boolean;@@ -42,6 +43,7 @@ export async function scrapeDocument_F0(if (isUrlBlocked(options.url, options.flags ?? null, {team_id: options.teamId,+ org_id: options.orgId ?? null,origin: options.origin,})) {@@ -68,6 +70,7 @@ export async function scrapeDocument_F0(scrapeOptions,internalOptions: {teamId: options.teamId,+ orgId: options.orgId ?? null,bypassBilling: true,threatProtection: options.threatProtectionPolicy ?? undefined,},diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.tsindex 3613903ff..553be8a86 100644--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts@@ -180,6 +180,7 @@ export async function performExtraction_F0(url,prompt: request.prompt,teamId,+ orgId: acuc?.org_id ?? null,allowExternalLinks: request.allowExternalLinks,origin: request.origin,limit: request.limit,@@ -348,6 +349,7 @@ export async function performExtraction_F0({url,teamId,+ orgId: acuc?.org_id ?? null,origin: "extract",timeout,flags: acuc?.flags ?? null,@@ -644,6 +646,7 @@ export async function performExtraction_F0({url,teamId,+ orgId: acuc?.org_id ?? null,origin: "extract",timeout,flags: acuc?.flags ?? null,diff --git a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts b/apps/api/src/lib/extract/fire-0/url-processor-f0.tsindex 650288a46..c97b2f636 100644--- a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts+++ b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts@@ -53,6 +53,7 @@ interface ProcessUrlOptions {prompt?: string;schema?: any;teamId: string;+ orgId?: string | null;allowExternalLinks?: boolean;origin?: string;limit?: number;@@ -80,6 +81,7 @@ export async function processUrl_F0(if (!isUrlBlocked(options.url, teamFlags, {team_id: options.teamId,+ org_id: options.orgId ?? null,origin: options.origin ?? null,})) {@@ -117,6 +119,7 @@ export async function processUrl_F0(url: baseUrl,search: searchQuery,teamId: options.teamId,+ orgId: options.orgId ?? null,allowExternalLinks: options.allowExternalLinks,origin: options.origin,limit: options.limit,@@ -154,6 +157,7 @@ export async function processUrl_F0(const retryMapResults = await getMapResults({url: baseUrl,teamId: options.teamId,+ orgId: options.orgId ?? null,allowExternalLinks: options.allowExternalLinks,origin: options.origin,limit: options.limit,diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.tsindex 71da9bb21..f841c2c82 100644--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts@@ -3,6 +3,7 @@ import { updateGeneratedLlmsTxt } from "./generate-llmstxt-redis";import { getMapResults } from "../../controllers/v1/map";import { z } from "zod";import { scrapeDocument } from "../extract/document-scraper";+import { hasOrgScopedBlocklist } from "../../scraper/WebScraper/utils/blocklist";import {getLlmsTextFromCache,saveLlmsTextToCache,@@ -87,10 +88,18 @@ export async function performGenerateLlmsTxt(// Enforce max URL limitconst effectiveMaxUrls = Math.min(maxUrls, 5000);+ // The cache is shared across teams, so a requester under org-scoped+ // blocklist rules must not touch it at all: even with the origin URL+ // itself unblocked, mapped child URLs may still be blocked, so a cached+ // generation could hand them blocked content and their filtered output+ // would poison the cache for everyone else.+ const skipSharedCache = hasOrgScopedBlocklist(acuc?.org_id);+// Check cache first, unless cache is set to false- const cachedResult = cache- ? await getLlmsTextFromCache(url, effectiveMaxUrls)- : null;+ const cachedResult =+ cache && !skipSharedCache+ ? await getLlmsTextFromCache(url, effectiveMaxUrls)+ : null;if (cachedResult) {logger.info("Found cached LLMs text", { url });@@ -130,6 +139,7 @@ export async function performGenerateLlmsTxt(const mapResult = await getMapResults({url,teamId,+ orgId: acuc?.org_id ?? null,limit: effectiveMaxUrls,includeSubdomains: false,ignoreSitemap: false,@@ -159,6 +169,7 @@ export async function performGenerateLlmsTxt({url,teamId,+ orgId: acuc?.org_id ?? null,origin: "llmstxt",timeout: 30000,isSingleUrl: true,@@ -233,7 +244,9 @@ export async function performGenerateLlmsTxt(}// After successful generation, save to cache- await saveLlmsTextToCache(url, llmstxt, llmsFulltxt, effectiveMaxUrls);+ if (!skipSharedCache) {+ await saveLlmsTextToCache(url, llmstxt, llmsFulltxt, effectiveMaxUrls);+ }// Limit pages and remove separators before final updateconst limitedFullText = limitPages(llmsFulltxt, effectiveMaxUrls);diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.tsindex 43bbc8b66..056d4a7cc 100644--- a/apps/api/src/lib/map-utils.ts+++ b/apps/api/src/lib/map-utils.ts@@ -86,6 +86,7 @@ export async function getMapResults({includeSubdomains = true,crawlerOptions = {},teamId,+ orgId,allowExternalLinks,abort = new AbortController().signal,filterByPath = true,@@ -103,6 +104,7 @@ export async function getMapResults({includeSubdomains?: boolean;crawlerOptions?: any;teamId: string;+ orgId?: string | null;origin?: string;includeMetadata?: boolean;allowExternalLinks?: boolean;@@ -144,7 +146,7 @@ export async function getMapResults({...(location ? { location } : {}),...(headers ? { headers } : {}),}),- internalOptions: { teamId },+ internalOptions: { teamId, orgId: orgId ?? null },team_id: teamId,createdAt: Date.now(),zeroDataRetention,@@ -370,6 +372,7 @@ export async function buildPromptWithWebsiteStructure({basePrompt,url,teamId,+ orgId,flags,logger,limit = 50,@@ -381,6 +384,7 @@ export async function buildPromptWithWebsiteStructure({basePrompt: string;url: string;teamId: string;+ orgId?: string | null;flags: TeamFlags | null;logger: Logger;limit?: number;@@ -397,6 +401,7 @@ export async function buildPromptWithWebsiteStructure({includeSubdomains,crawlerOptions: { sitemap: "include" },teamId,+ orgId: orgId ?? null,flags,allowExternalLinks,filterByPath: false,diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsindex 1b27a8c0a..4e8f5e00f 100644--- a/apps/api/src/lib/robots-txt.ts+++ b/apps/api/src/lib/robots-txt.ts@@ -79,6 +79,7 @@ export async function fetchRobotsTxt({forceEngine,v0DisableJsDom: true,+ orgId: null,externalAbort: abort? {signal: abort,diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.tsindex 9c1f9e768..fd9cc5a41 100644--- a/apps/api/src/lib/scrape-billing.test.ts+++ b/apps/api/src/lib/scrape-billing.test.ts@@ -13,6 +13,7 @@ describe("calculateCreditsToBeBilled", () => {} as any,{teamId: "team-id",+ orgId: null,},{metadata: {@@ -40,6 +41,7 @@ describe("calculateCreditsToBeBilled", () => {} as any,{teamId: "team-id",+ orgId: null,},{metadata: {@@ -64,6 +66,7 @@ describe("calculateCreditsToBeBilled", () => {} as any,{teamId: "team-id",+ orgId: null,},{metadata: {@@ -95,6 +98,7 @@ describe("calculateCreditsToBeBilled", () => {} as any,{teamId: "team-id",+ orgId: null,},{metadata: {@@ -163,7 +167,7 @@ const billWithDecisions = (args: {}) =>calculateCreditsToBeBilled({ formats: [{ type: "markdown" }] } as any,- { teamId: "team-id" },+ { teamId: "team-id", orgId: null },args.document as any,{ totalCost: 0 } as any,{} as any,diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex c80c7dd3b..c52d15f8f 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -29,6 +29,8 @@ export async function startWebScraperPipeline({crawlId: job.data.crawl_id,teamId: job.data.team_id,...job.data.internalOptions,+ // In-flight jobs enqueued before orgId existed may omit it.+ orgId: job.data.internalOptions?.orgId ?? null,},team_id: job.data.team_id,bull_job_id: job.id,@@ -89,6 +91,7 @@ async function runWebScraper({...internalOptions,urlInvisibleInCurrentCrawl,teamId: internalOptions?.teamId ?? team_id,+ orgId: internalOptions?.orgId ?? null,},costTracking,);diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 73f553e3b..d3e78aae9 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -341,6 +341,7 @@ function blocklistGate(!canUseExchange &&isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {team_id: req.acuc?.team_id ?? null,+ org_id: req.acuc?.org_id ?? null,origin: typeof req.body.origin === "string" ? req.body.origin : null,})) {diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 81824a3d0..62d240bed 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -115,6 +115,7 @@ export async function getLinksFromSitemap({forceEngine,v0DisableJsDom: true,+ orgId: null,externalAbort: abort? {signal: abort,diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.test.tsnew file mode 100644index 000000000..c290111ac--- /dev/null+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts@@ -0,0 +1,223 @@+import type { TeamFlags } from "../../../controllers/v1/types";+import {+ hasOrgScopedBlocklist,+ initializeBlocklist,+ isUrlBlocked,+} from "./blocklist";++const dbState = vi.hoisted(() => ({+ rows: [] as { data: unknown; org_id: string | null }[],+}));++vi.mock("../../../config", () => ({+ config: {+ USE_DB_AUTHENTICATION: true,+ DISABLE_BLOCKLIST: false,+ },+}));++vi.mock("../../../db/connection", () => ({+ db: {+ insert: vi.fn(() => ({ values: vi.fn(async () => {}) })),+ },+ dbRr: {+ select: vi.fn(() => ({+ from: vi.fn(async () => dbState.rows),+ })),+ },+}));++const ORG_A = "00000000-0000-0000-0000-00000000000a";+const ORG_B = "00000000-0000-0000-0000-00000000000b";+const TEAM = "00000000-0000-0000-0000-000000000001";++// NOTE: the matcher also blocks other TLDs of the same base domain, so+// placeholder domains here must not share a base name across lists unless+// the test is exercising exactly that rule.+const GLOBAL_ROW = {+ org_id: null,+ data: {+ blocklist: ["facebook.com"],+ allowedKeywords: ["/legal"],+ },+};++function ctx(org_id: string | null) {+ return { team_id: TEAM, org_id, origin: "test" };+}++describe("isUrlBlocked", () => {+ describe("global entries", () => {+ beforeAll(async () => {+ dbState.rows = [GLOBAL_ROW];+ await initializeBlocklist();+ });++ it("blocks a global entry with no context", () => {+ expect(isUrlBlocked("https://facebook.com/profile", null)).toBe(true);+ });++ it("blocks a global entry for any org", () => {+ expect(+ isUrlBlocked("https://facebook.com/profile", null, ctx(ORG_A)),+ ).toBe(true);+ expect(+ isUrlBlocked("https://facebook.com/profile", null, ctx(null)),+ ).toBe(true);+ });++ it("does not block unlisted domains", () => {+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_A))).toBe(false);+ });++ it("respects global allowedKeywords", () => {+ expect(isUrlBlocked("https://facebook.com/legal", null)).toBe(false);+ });++ it("unblocks a global entry via flags.unblockedDomains", () => {+ const flags = { unblockedDomains: ["facebook.com"] } as TeamFlags;+ expect(isUrlBlocked("https://facebook.com/profile", flags)).toBe(false);+ });+ });++ describe("org-scoped entries", () => {+ beforeAll(async () => {+ dbState.rows = [+ GLOBAL_ROW,+ {+ org_id: ORG_A,+ data: {+ blocklist: ["example.com", "org-blocked.com"],+ allowedKeywords: ["/allowed-path"],+ },+ },+ ];+ await initializeBlocklist();+ });++ it("blocks the org's entry for that org only", () => {+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_A))).toBe(true);+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_B))).toBe(false);+ expect(isUrlBlocked("https://example.com", null, ctx(null))).toBe(false);+ expect(isUrlBlocked("https://example.com", null)).toBe(false);+ });++ it("keeps blocking global entries for the org", () => {+ expect(+ isUrlBlocked("https://facebook.com/profile", null, ctx(ORG_A)),+ ).toBe(true);+ });++ it("blocks subdomains of an org-scoped entry", () => {+ expect(+ isUrlBlocked("https://sub.example.com/page", null, ctx(ORG_A)),+ ).toBe(true);+ });++ it("blocks different TLDs of an org-scoped entry", () => {+ expect(isUrlBlocked("https://example.de", null, ctx(ORG_A))).toBe(true);+ });++ it("unblocks org-scoped entries via flags.unblockedDomains", () => {+ const flags = { unblockedDomains: ["example.com"] } as TeamFlags;+ expect(isUrlBlocked("https://example.com", flags, ctx(ORG_A))).toBe(+ false,+ );+ // Only the listed domain is unblocked, not the org's other entries.+ expect(isUrlBlocked("https://org-blocked.com", flags, ctx(ORG_A))).toBe(+ true,+ );+ });++ it("respects the org blob's own allowedKeywords for its list only", () => {+ expect(+ isUrlBlocked("https://example.com/allowed-path", null, ctx(ORG_A)),+ ).toBe(false);+ // The org's allowedKeywords do not exempt globally-blocked URLs.+ expect(+ isUrlBlocked("https://facebook.com/allowed-path", null, ctx(ORG_A)),+ ).toBe(true);+ });++ it("reports which orgs have org-scoped entries", () => {+ expect(hasOrgScopedBlocklist(ORG_A)).toBe(true);+ expect(hasOrgScopedBlocklist(ORG_B)).toBe(false);+ expect(hasOrgScopedBlocklist(null)).toBe(false);+ expect(hasOrgScopedBlocklist(undefined)).toBe(false);+ });++ it("does not count an org whose rows hold no blockable entries", async () => {+ dbState.rows = [+ GLOBAL_ROW,+ { org_id: ORG_B, data: { blocklist: [], allowedKeywords: ["/x"] } },+ ];+ await initializeBlocklist();+ expect(hasOrgScopedBlocklist(ORG_B)).toBe(false);+ });++ it("does not count an org whose entries are all blank strings", async () => {+ dbState.rows = [+ GLOBAL_ROW,+ {+ org_id: ORG_B,+ data: { blocklist: ["", " "], allowedKeywords: [] },+ },+ ];+ await initializeBlocklist();+ expect(hasOrgScopedBlocklist(ORG_B)).toBe(false);+ });+ });++ describe("row loading", () => {+ it("merges multiple rows for the same org", async () => {+ dbState.rows = [+ GLOBAL_ROW,+ {+ org_id: ORG_A,+ data: { blocklist: ["example-one.com"], allowedKeywords: [] },+ },+ {+ org_id: ORG_A,+ data: { blocklist: ["example-two.com"], allowedKeywords: [] },+ },+ ];+ await initializeBlocklist();++ expect(isUrlBlocked("https://example-one.com", null, ctx(ORG_A))).toBe(+ true,+ );+ expect(isUrlBlocked("https://example-two.com", null, ctx(ORG_A))).toBe(+ true,+ );+ });++ it("tolerates malformed org row data without breaking loading", async () => {+ dbState.rows = [+ GLOBAL_ROW,+ { org_id: ORG_A, data: null },+ { org_id: ORG_B, data: { blocklist: "not-an-array" } },+ ];+ await initializeBlocklist();++ expect(isUrlBlocked("https://facebook.com/profile", null)).toBe(true);+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_A))).toBe(false);+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_B))).toBe(false);+ });++ it("throws when the table is empty", async () => {+ dbState.rows = [];+ await expect(initializeBlocklist()).rejects.toThrow(+ "No data returned from database",+ );+ });++ it("throws when rows exist but none is global", async () => {+ dbState.rows = [+ { org_id: ORG_A, data: { blocklist: [], allowedKeywords: [] } },+ ];+ await expect(initializeBlocklist()).rejects.toThrow(+ "No global blocklist row",+ );+ });+ });+});diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.tsindex eabf66ed1..04a758b55 100644--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts@@ -15,6 +15,7 @@ type BlocklistBlob = {type BlockContext = {team_id?: string | null;+ org_id?: string | null;origin?: string | null;};@@ -102,6 +103,40 @@ function allowedKeywordMatches(url: string, allowedKeyword: string): boolean {}let blob: BlocklistBlob | null = null;+let orgBlobs: Map<string, BlocklistBlob> = new Map();++// Rows are written by ops directly, so reads must survive a missing or+// partial document — a bad row must never take down startup. Blank entries+// can never match and would make an org register as scoped, so drop them.+function parseStringEntries(value: unknown): string[] {+ return Array.isArray(value)+ ? value.filter(+ (x): x is string => typeof x === "string" && x.trim().length > 0,+ )+ : [];+}++function parseBlob(data: unknown): BlocklistBlob {+ const doc = (data ?? {}) as Record<string, unknown>;+ return {+ blocklist: parseStringEntries(doc.blocklist),+ allowedKeywords: parseStringEntries(doc.allowedKeywords),+ };+}++function mergeBlob(target: BlocklistBlob, source: BlocklistBlob): void {+ target.blocklist.push(...source.blocklist);+ target.allowedKeywords.push(...source.allowedKeywords);+}++// Rows may repeat entries; the match loops scan linearly, so dedupe once at+// load instead of paying for duplicates on every check.+function dedupeBlob(blob: BlocklistBlob): BlocklistBlob {+ return {+ blocklist: [...new Set(blob.blocklist)],+ allowedKeywords: [...new Set(blob.allowedKeywords)],+ };+}export async function initializeBlocklist() {if (config.USE_DB_AUTHENTICATION !== true || config.DISABLE_BLOCKLIST) {@@ -109,79 +144,96 @@ export async function initializeBlocklist() {blocklist: [],allowedKeywords: [],};+ orgBlobs = new Map();return;}- let data: { data: any } | undefined;+ let rows: { data: any; org_id: string | null }[];try {- [data] = await dbRr.select().from(schema.blocklist).limit(1);+ rows = await dbRr.select().from(schema.blocklist);} catch (error) {throw new Error(`Error getting blocklist: ${error instanceof Error ? error.message : JSON.stringify(error)}`,);}- if (!data) {+ if (rows.length === 0) {throw new Error("Error getting blocklist: No data returned from database");}- blob = data.data;-}-export function isUrlBlocked(- url: string,- flags: TeamFlags,- context?: BlockContext,-): boolean {- if (blob === null) {- throw new Error("Blocklist not initialized");+ if (!rows.some(row => row.org_id === null)) {+ throw new Error(+ "Error getting blocklist: No global blocklist row (org_id IS NULL) found",+ );}- const lowerCaseUrl = url.trim().toLowerCase();-- let blockedlist = [...blob.blocklist];-- if (flags?.unblockedDomains) {- blockedlist = blockedlist.filter(- blocked => !flags.unblockedDomains!.includes(blocked),- );+ const globalBlob: BlocklistBlob = { blocklist: [], allowedKeywords: [] };+ const perOrg = new Map<string, BlocklistBlob>();+ for (const row of rows) {+ if (row.org_id === null) {+ mergeBlob(globalBlob, parseBlob(row.data));+ } else {+ const existing = perOrg.get(row.org_id);+ if (existing) {+ mergeBlob(existing, parseBlob(row.data));+ } else {+ perOrg.set(row.org_id, parseBlob(row.data));+ }+ }}+ for (const [orgId, orgBlob] of perOrg) {+ const deduped = dedupeBlob(orgBlob);+ // An org whose rows hold no blockable entries must not register as+ // org-scoped at all — hasOrgScopedBlocklist consumers would otherwise+ // pay cache opt-outs for an org that can never block anything.+ if (deduped.blocklist.length === 0) {+ perOrg.delete(orgId);+ } else {+ perOrg.set(orgId, deduped);+ }+ }+ blob = dedupeBlob(globalBlob);+ orgBlobs = perOrg;+}+function findBlockedMatch(+ url: string,+ lowerCaseUrl: string,+ blockedlist: string[],+ allowedKeywords: string[],+): string | null {const decryptedUrl =blockedlist.find(decrypted => lowerCaseUrl === decrypted) || lowerCaseUrl;- // If the URL is empty or invalid, return false+ // If the URL is empty or invalid, return no matchlet parsedUrl: any;try {parsedUrl = parse(decryptedUrl);} catch {console.log("Error parsing URL:", url);- return false;+ return null;}const domain = parsedUrl.domain;const publicSuffix = parsedUrl.publicSuffix;if (!domain) {- return false;+ return null;}// Check if URL contains any allowed keyword- if (- blob.allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))- ) {- return false;+ if (allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))) {+ return null;}// Block exact matchesif (blockedlist.includes(domain)) {- recordHit(url, domain, context);- return true;+ return domain;}// Block subdomainsif (blockedlist.some(blocked => domain.endsWith(`.${blocked}`))) {- recordHit(url, domain, context);- return true;+ return domain;}// Block different TLDs of the same base domain@@ -194,9 +246,76 @@ export function isUrlBlocked(blocked => blocked.startsWith(baseDomain + ".") && blocked !== domain,)) {- recordHit(url, domain, context);+ return domain;+ }++ return null;+}++/**+ * Whether the org has any org-scoped blocklist entries at all. Lets shared+ * caches opt such requesters out entirely: a per-URL check is not enough+ * when the cached artifact covers other URLs than the one checked.+ */+export function hasOrgScopedBlocklist(+ orgId: string | null | undefined,+): boolean {+ if (blob === null) {+ throw new Error("Blocklist not initialized");+ }+ return typeof orgId === "string" && orgBlobs.has(orgId);+}++export function isUrlBlocked(+ url: string,+ flags: TeamFlags,+ context?: BlockContext,+): boolean {+ if (blob === null) {+ throw new Error("Blocklist not initialized");+ }++ const lowerCaseUrl = url.trim().toLowerCase();++ let blockedlist = [...blob.blocklist];++ if (flags?.unblockedDomains) {+ blockedlist = blockedlist.filter(+ blocked => !flags.unblockedDomains!.includes(blocked),+ );+ }++ const globalMatch = findBlockedMatch(+ url,+ lowerCaseUrl,+ blockedlist,+ blob.allowedKeywords,+ );+ if (globalMatch !== null) {+ recordHit(url, globalMatch, context);return true;}+ // The org blob's own allowedKeywords exempt URLs from the org list only.+ const orgBlob = context?.org_id ? orgBlobs.get(context.org_id) : undefined;+ if (orgBlob) {+ let orgBlockedlist = orgBlob.blocklist;+ if (flags?.unblockedDomains) {+ orgBlockedlist = orgBlockedlist.filter(+ blocked => !flags.unblockedDomains!.includes(blocked),+ );+ }+ const orgMatch = findBlockedMatch(+ url,+ lowerCaseUrl,+ orgBlockedlist,+ orgBlob.allowedKeywords,+ );+ if (orgMatch !== null) {+ recordHit(url, orgMatch, context);+ return true;+ }+ }+return false;}diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsindex dfa68556e..040de8599 100644--- a/apps/api/src/scraper/crawler/sitemap.ts+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -88,6 +88,7 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {v0DisableJsDom: true,// externalAbort: options.abort,teamId: "sitemap",+ orgId: null,zeroDataRetention: options.zeroDataRetention,crawlId: options.crawlId,isPreCrawl: options.isPreCrawl,diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 60b986bc7..9660a4584 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -7,10 +7,7 @@ import {scrapeURLWithFireEngineChromeCDP,scrapeURLWithFireEngineTLSClient,} from "./fire-engine";-import {- exchangeMaxReasonableTime,- scrapeURLWithExchange,-} from "./exchange";+import { exchangeMaxReasonableTime, scrapeURLWithExchange } from "./exchange";import { pdfMaxReasonableTime, scrapePDF } from "./pdf";import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";import {@@ -178,7 +175,7 @@ export type EngineScrapeResult = {const engineHandlers: {[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;} = {- "exchange": scrapeURLWithExchange,+ exchange: scrapeURLWithExchange,index: scrapeURLWithIndex,"index;documents": scrapeURLWithIndex,"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,@@ -198,7 +195,7 @@ const engineHandlers: {const engineMRTs: {[E in Engine]: (meta: Meta) => number;} = {- "exchange": exchangeMaxReasonableTime,+ exchange: exchangeMaxReasonableTime,index: indexMaxReasonableTime,"index;documents": indexMaxReasonableTime,"fire-engine;chrome-cdp": meta =>@@ -231,7 +228,7 @@ const engineOptions: {quality: number;};} = {- "exchange": {+ exchange: {features: {actions: false,waitFor: false,@@ -632,6 +629,7 @@ export async function buildFallbackList(meta: Meta): Promise<meta.internalOptions.teamFlags ?? null,{team_id: meta.internalOptions.teamId ?? null,+ org_id: meta.internalOptions.orgId ?? null,origin: null,},)@@ -639,10 +637,9 @@ export async function buildFallbackList(meta: Meta): Promise<return [];}} catch (error) {- meta.logger.warn(- "Exchange blocklist re-check failed; failing closed",- { error },- );+ meta.logger.warn("Exchange blocklist re-check failed; failing closed", {+ error,+ });return [];}}diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 904bd11cf..70be2d8de 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -493,6 +493,12 @@ export type InternalOptions = {bypassBilling?: boolean;zeroDataRetention?: boolean;teamFlags?: TeamFlags;+ /** Team's org, snapshotted from the request ACUC at acceptance (same+ * pattern as teamFlags). Rides the job payload so org-scoped blocklist+ * checks work without re-fetching the chunk. Required so a payload+ * builder cannot silently omit the org and skip org-scoped enforcement;+ * pass null when the caller genuinely has no org (internal/system work). */+ orgId: string | null;/** Team's sold concurrency, snapshotted from the request ACUC at* acceptance (same pattern as teamFlags). Rides the job payload so* downstream engines (FirePDF async account context) never re-fetch. */diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.tsindex e542cc93f..c5d4a6d9a 100644--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts@@ -10,7 +10,7 @@ import { Engine } from "./engines";import { CostTracking } from "../../lib/cost-tracking";// Mock parseMarkdown but delegate to real implementation for other tests-vi.mock("../../lib/html-to-markdown", async (importOriginal) => {+vi.mock("../../lib/html-to-markdown", async importOriginal => {const actual =await importOriginal<typeof import("../../lib/html-to-markdown")>();return {@@ -40,7 +40,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-basic","https://firecrawl-test-site.vercel.app",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -79,7 +79,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["markdown", "html"],}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -104,7 +104,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({onlyMainContent: false,}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -128,7 +128,7 @@ describe("Standalone scrapeURL tests", () => {onlyMainContent: false,excludeTags: [".nav", "#footer", "strong"],}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -149,7 +149,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-400","https://httpstat.us/400",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -168,7 +168,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-401","https://httpstat.us/401",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -187,7 +187,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-403","https://httpstat.us/403",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -206,7 +206,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-404","https://httpstat.us/404",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -225,7 +225,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-405","https://httpstat.us/405",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -244,7 +244,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-500","https://httpstat.us/500",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -263,7 +263,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-redirect","https://scrapethissite.com/",scrapeOptions.parse({}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -296,7 +296,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["screenshot"],}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -325,7 +325,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["screenshot@fullPage"],}),- { forceEngine, teamId: "test" },+ { forceEngine, teamId: "test", orgId: null },new CostTracking(),);@@ -354,7 +354,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-pdf","https://arxiv.org/pdf/astro-ph/9301001.pdf",scrapeOptions.parse({}),- { teamId: "test" },+ { teamId: "test", orgId: null },new CostTracking(),);@@ -374,7 +374,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-docx","https://nvca.org/wp-content/uploads/2019/06/NVCA-Model-Document-Stock-Purchase-Agreement.docx",scrapeOptions.parse({}),- { teamId: "test" },+ { teamId: "test", orgId: null },new CostTracking(),);@@ -396,7 +396,7 @@ describe("Standalone scrapeURL tests", () => {"test:scrape-xlsx","https://download.microsoft.com/download/1/4/E/14EDED28-6C58-4055-A65C-23B4DA81C4DE/Financial%20Sample.xlsx",scrapeOptions.parse({}),- { teamId: "test" },+ { teamId: "test", orgId: null },new CostTracking(),);@@ -437,7 +437,7 @@ describe("Standalone scrapeURL tests", () => {},},}),- { teamId: "test" },+ { teamId: "test", orgId: null },new CostTracking(),);@@ -474,7 +474,7 @@ describe("Standalone scrapeURL tests", () => {},},}),- { teamId: "test" },+ { teamId: "test", orgId: null },new CostTracking(),);@@ -501,7 +501,7 @@ describe("Standalone scrapeURL tests", () => {id,url,scrapeOptions.parse({}),- { teamId: "test" },+ { teamId: "test", orgId: null },new CostTracking(),);@@ -545,7 +545,7 @@ describe("Standalone scrapeURL tests", () => {scrapeOptions.parse({formats: ["rawHtml"],}),- { teamId: "sitemap" },+ { teamId: "sitemap", orgId: null },new CostTracking(),);diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 5cab96b50..15f5186ef 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -46,6 +46,7 @@ interface SearchOptions {interface SearchContext {teamId: string;+ orgId?: string | null;origin: string;integration?: string | null;apiKeyId: number | null;@@ -79,6 +80,7 @@ export async function executeSearch(const { query, limit, sources, categories, scrapeOptions } = options;const {teamId,+ orgId,origin,apiKeyId,flags,@@ -212,12 +214,14 @@ export async function executeSearch(if (shouldScrape && scrapeOptions) {const itemsToScrape = getItemsToScrape(searchResponse, flags, {team_id: teamId,+ org_id: orgId ?? null,origin,});if (itemsToScrape.length > 0) {const scrapeOpts = {teamId,+ orgId: orgId ?? null,origin,timeout: options.timeout,scrapeOptions,diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.tsindex d14f4a3cb..5ac8c353e 100644--- a/apps/api/src/search/scrape.ts+++ b/apps/api/src/search/scrape.ts@@ -31,6 +31,7 @@ interface ScrapeItem {interface ScrapeSearchOptions {teamId: string;+ orgId?: string | null;origin: string;timeout: number;scrapeOptions: ScrapeOptions;@@ -79,6 +80,7 @@ async function scrapeSearchResultDirect(},internalOptions: {teamId: options.teamId,+ orgId: options.orgId ?? null,bypassBilling: options.bypassBilling ?? true,zeroDataRetention,teamFlags: flags,@@ -147,7 +149,11 @@ async function scrapeSearchResultDirect(export function getItemsToScrape(searchResponse: SearchV2Response,flags: TeamFlags,- context?: { team_id?: string | null; origin?: string | null },+ context?: {+ team_id?: string | null;+ org_id?: string | null;+ origin?: string | null;+ },): ScrapeItem[] {const items: ScrapeItem[] = [];diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex cd671fd62..4858328ab 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -503,6 +503,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {internalOptions: {disableSmartWaitCache: true, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matterteamId,+ orgId: null, // internal pre-crawl team, no org appliessaveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,zeroDataRetention: false,isPreCrawl: true, // NOTE: must be added to internal options for indexing, if not it will be treated as a normal scrape in the indexdiff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 3b5849909..c3cd12dfe 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -303,6 +303,9 @@ async function scrapeSearchMonitorPage(params: {scrapeOptions,internalOptions: {teamId: params.teamId,+ // Monitors do no in-pipeline blocklist enforcement (business rule);+ // search results are filtered via isBlocked before scraping.+ orgId: null,saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,bypassBilling: true,zeroDataRetention: false,@@ -535,6 +538,8 @@ async function enqueueMonitorScrapeTarget(params: {scrapeOptions,internalOptions: {teamId: params.monitor.team_id,+ // Monitors do no in-pipeline blocklist enforcement (business rule).+ orgId: null,saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,bypassBilling: true,zeroDataRetention: false,@@ -603,6 +608,8 @@ async function enqueueMonitorCrawlTarget(params: {internalOptions: {disableSmartWaitCache: true,teamId: params.monitor.team_id,+ // Monitors do no in-pipeline blocklist enforcement (business rule).+ orgId: null,saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,zeroDataRetention: false,bypassBilling: true,@@ -778,6 +785,7 @@ async function runMonitorSearchTarget(params: {isBlocked: url =>isUrlBlocked(url, teamFlags, {team_id: monitor.team_id,+ org_id: acuc?.org_id ?? null,origin: "monitor.search",}),goalVersion,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex ac40ba261..bfabe4ead 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -464,15 +464,13 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {await saveCrawl(job.data.crawl_id, sc);}+ const teamChunk = await getACUCTeam(job.data.team_id);if (- isUrlBlocked(- doc.metadata.url,- (await getACUCTeam(job.data.team_id))?.flags ?? null,- {- team_id: job.data.team_id,- origin: job.data.origin,- },- )+ isUrlBlocked(doc.metadata.url, teamChunk?.flags ?? null, {+ team_id: job.data.team_id,+ org_id: teamChunk?.org_id ?? null,+ origin: job.data.origin,+ })) {throw new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE); // TODO: make this its own error type that is ignored by error tracking}