Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): thread org context through the scrape pipeline (#4064)

Rafael Miller93387da4fbdf
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index d050b99d3..9ce2cb5ab 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -77,6 +77,7 @@ const mockPreviewACUC: (
api_key: "preview",
api_key_id: 0,
team_id,
+ org_id: "preview",
rate_limits: {
crawl: 2,
scrape: 10,
@@ -102,6 +103,7 @@ const mockACUC: () => AuthCreditUsageChunk = () => ({
api_key: "bypass",
api_key_id: 0,
team_id: "bypass",
+ org_id: "bypass",
rate_limits: {
crawl: 99999999,
scrape: 99999999,
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
index a1a9c4682..75ba03b18 100644
--- a/apps/api/src/controllers/v0/admin/crawl-monitor.ts
+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
@@ -61,6 +61,7 @@ export async function crawlMonitorController(
internalOptions: {
disableSmartWaitCache: true,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention: false,
},
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index ee19ad91b..3cc6ec708 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -180,6 +180,7 @@ export async function crawlController(req: Request, res: Response) {
if (
isUrlBlocked(url, auth.chunk?.flags ?? null, {
team_id: auth.chunk?.team_id ?? team_id,
+ org_id: auth.chunk?.org_id ?? null,
origin: req.body?.origin ?? null,
})
) {
@@ -224,6 +225,7 @@ export async function crawlController(req: Request, res: Response) {
team_id,
);
internalOptions.disableSmartWaitCache = true; // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
+ internalOptions.orgId = auth.chunk?.org_id ?? null;
internalOptions.saveScrapeResultToGCS = process.env
.GCS_FIRE_ENGINE_BUCKET_NAME
? true
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 4679ed2a0..8ae9a1a7c 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -41,6 +41,7 @@ async function scrapeHelper(
extractorOptions: ExtractorOptions,
timeout: number,
flags: TeamFlags,
+ org_id: string | null,
apiKeyId: number | null,
): Promise<{
success: boolean;
@@ -70,6 +71,7 @@ async function scrapeHelper(
if (
isUrlBlocked(url, flags, {
team_id,
+ org_id,
origin: req.body?.origin ?? null,
})
) {
@@ -88,6 +90,7 @@ async function scrapeHelper(
team_id,
);
+ internalOptions.orgId = org_id;
internalOptions.saveScrapeResultToGCS = process.env
.GCS_FIRE_ENGINE_BUCKET_NAME
? true
@@ -299,6 +302,7 @@ export async function scrapeController(req: Request, res: Response) {
extractorOptions,
timeout,
chunk?.flags ?? null,
+ chunk?.org_id ?? null,
chunk?.api_key_id ?? null,
);
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 19463249b..55bd5ee8b 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -37,6 +37,7 @@ async function searchHelper(
pageOptions: PageOptions,
searchOptions: SearchOptions,
flags: TeamFlags,
+ org_id: string | null,
api_key_id: number | null,
): Promise<{
success: boolean;
@@ -81,6 +82,7 @@ async function searchHelper(
crawlerOptions,
team_id,
);
+ internalOptions.orgId = org_id;
if (justSearch) {
const searchCredits = Math.ceil(res.length / 10) * 2;
@@ -103,6 +105,7 @@ async function searchHelper(
r =>
!isUrlBlocked(r.url, flags, {
team_id,
+ org_id,
origin: req.body?.origin ?? null,
}),
);
@@ -269,6 +272,7 @@ export async function searchController(req: Request, res: Response) {
pageOptions,
searchOptions,
chunk?.flags ?? null,
+ chunk?.org_id ?? null,
chunk?.api_key_id ?? null,
);
const endTime = new Date().getTime();
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 8e5d4750a..d20906b9f 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -121,6 +121,7 @@ export async function batchScrapeController(
if (
!isUrlBlocked(nu, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
+ org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
})
) {
@@ -138,6 +139,7 @@ export async function batchScrapeController(
req.body.urls?.some((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
+ org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
}),
)
@@ -250,6 +252,7 @@ export async function batchScrapeController(
...internalOptions,
disableSmartWaitCache: true,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index fd730e57f..877cc43f8 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -201,6 +201,7 @@ export async function crawlController(
...internalOptions,
disableSmartWaitCache: true,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index 75b72462b..b1b778a31 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -124,6 +124,7 @@ export async function extractController(
req.body.urls?.filter((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
+ org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
}),
) ?? [];
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 052d88f0b..b9ec77ecd 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -93,6 +93,7 @@ export async function getMapResults({
includeSubdomains = true,
crawlerOptions = {},
teamId,
+ orgId,
origin,
includeMetadata = false,
allowExternalLinks,
@@ -114,6 +115,7 @@ export async function getMapResults({
includeSubdomains?: boolean;
crawlerOptions?: any;
teamId: string;
+ orgId?: string | null;
origin?: string;
includeMetadata?: boolean;
allowExternalLinks?: boolean;
@@ -146,7 +148,7 @@ export async function getMapResults({
...(location ? { location } : {}),
...(headers ? { headers } : {}),
}),
- internalOptions: { teamId },
+ internalOptions: { teamId, orgId: orgId ?? null },
team_id: teamId,
createdAt: Date.now(),
};
@@ -445,6 +447,7 @@ export async function mapController(
crawlerOptions: req.body,
origin: req.body.origin,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
abort: abort.signal,
mock: req.body.useMock,
filterByPath: req.body.filterByPath !== false,
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 512d8280b..d5bc60f09 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -227,6 +227,7 @@ export async function scrapeController(
bypassBilling: isDirectToBullMQ,
zeroDataRetention,
teamFlags: req.acuc?.flags ?? null,
+ orgId: req.acuc?.org_id ?? null,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
},
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 8656a4371..129b83e48 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -47,6 +47,7 @@ export async function searchAndScrapeSearchResult(
query: string,
options: {
teamId: string;
+ orgId?: string | null;
origin: string;
timeout: number;
scrapeOptions: any;
@@ -77,6 +78,7 @@ export async function searchAndScrapeSearchResult(
})),
{
teamId: options.teamId,
+ orgId: options.orgId ?? null,
origin: options.origin,
timeout: options.timeout,
scrapeOptions,
@@ -235,6 +237,7 @@ export async function searchController(
},
{
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
origin: req.body.origin,
integration: req.body.integration,
apiKeyId: req.acuc?.api_key_id ?? null,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 2284d4d45..4d9c63011 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1256,7 +1256,7 @@ export type AuthCreditUsageChunk = {
api_key: string;
api_key_id: number;
team_id: string;
- org_id?: string | null;
+ org_id: string;
plan_priority: {
bucketLimit: number;
planModifier: number;
@@ -1452,6 +1452,7 @@ function fromLegacyCrawlerOptions(
internalOptions: {
v0CrawlOnlyUrls: x.returnOnlyUrls,
teamId,
+ orgId: null,
},
};
}
@@ -1517,6 +1518,7 @@ export function fromLegacyScrapeOptions(
atsv: pageOptions.atsv,
v0DisableJsDom: pageOptions.disableJsDom,
teamId,
+ orgId: null,
},
// TODO: fallback, fetchPageContent, replaceAllPathsWithAbsolutePaths, includeLinks
};
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 7228d6921..d393df0e5 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -142,6 +142,7 @@ export async function batchScrapeController(
if (
!isUrlBlocked(nu, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
+ org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
})
) {
@@ -159,6 +160,7 @@ export async function batchScrapeController(
req.body.urls?.some((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
+ org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
}),
)
@@ -271,6 +273,7 @@ export async function batchScrapeController(
internalOptions: {
disableSmartWaitCache: true,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
diff --git a/apps/api/src/controllers/v2/crawl-cancel.ts b/apps/api/src/controllers/v2/crawl-cancel.ts
index f3cfe561a..a304860ac 100644
--- a/apps/api/src/controllers/v2/crawl-cancel.ts
+++ b/apps/api/src/controllers/v2/crawl-cancel.ts
@@ -41,6 +41,7 @@ export async function crawlCancelController(
scrapeOptions: scrapeOptions.parse({}),
internalOptions: {
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
},
};
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index 312a3af9a..fb7539f97 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -67,6 +67,7 @@ export async function crawlParamsPreviewController(
basePrompt: parsedBody.prompt,
url: parsedBody.url,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
logger,
limit: 50,
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 5fc1761ce..a0f1f9b1e 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -170,6 +170,7 @@ export async function crawlController(
basePrompt: req.body.prompt,
url: req.body.url,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
flags: req.acuc?.flags ?? null,
logger,
limit: 50,
@@ -262,6 +263,7 @@ export async function crawlController(
internalOptions: {
disableSmartWaitCache: true,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
zeroDataRetention,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index 59476f464..fb21af95f 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -67,6 +67,7 @@ export async function extractController(
req.body.urls?.filter((url: string) =>
isUrlBlocked(url, req.acuc?.flags ?? null, {
team_id: req.auth.team_id,
+ org_id: req.acuc?.org_id ?? null,
origin: req.body.origin ?? null,
}),
) ?? [];
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index b85681fb2..eb02d2ae9 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -167,6 +167,7 @@ export async function mapController(
},
origin: req.body.origin,
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
allowExternalLinks: req.body.allowExternalLinks,
abort: abort.signal,
mock: req.body.useMock,
diff --git a/apps/api/src/controllers/v2/parse.ts b/apps/api/src/controllers/v2/parse.ts
index f47a15917..9e879c2c3 100644
--- a/apps/api/src/controllers/v2/parse.ts
+++ b/apps/api/src/controllers/v2/parse.ts
@@ -512,6 +512,7 @@ export async function parseController(
bypassBilling: isDirectToBullMQ || !shouldBill,
zeroDataRetention,
teamFlags: req.acuc?.flags ?? null,
+ orgId: req.acuc?.org_id ?? null,
teamConcurrency: req.acuc?.concurrency ?? null,
uploadedFile: file,
forceEngine,
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 57b22730b..3ce6e1f77 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -328,6 +328,7 @@ export async function scrapeController(
bypassBilling: isDirectToBullMQ || !shouldBill,
zeroDataRetention,
teamFlags: req.acuc?.flags ?? null,
+ orgId: req.acuc?.org_id ?? null,
teamConcurrency: req.acuc?.concurrency ?? null,
agentIndexOnly: (req as any).agentIndexOnly ?? false,
threatProtection: threatProtection.policy ?? undefined,
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index f52f45fd7..55dba5ad6 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -218,6 +218,7 @@ export async function searchController(
},
{
teamId: req.auth.team_id,
+ orgId: req.acuc?.org_id ?? null,
origin: req.body.origin,
integration: req.body.integration,
apiKeyId: req.acuc?.api_key_id ?? null,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index f5e15000f..e73875a29 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1663,6 +1663,7 @@ function fromV0CrawlerOptions(
internalOptions: {
v0CrawlOnlyUrls: x.returnOnlyUrls,
teamId,
+ orgId: null,
},
};
}
@@ -1727,6 +1728,7 @@ export function fromV0ScrapeOptions(
atsv: pageOptions.atsv,
v0DisableJsDom: pageOptions.disableJsDom,
teamId,
+ orgId: null,
...(extractorOptions !== undefined &&
extractorOptions.mode.includes("llm-extraction")
? {
@@ -1839,6 +1841,7 @@ export function fromV1ScrapeOptions(
}),
internalOptions: {
teamId,
+ orgId: null,
v1Agent: v1ScrapeOptions.agent,
v1JSONSystemPrompt: (
v1ScrapeOptions.jsonOptions || v1ScrapeOptions.extract
diff --git a/apps/api/src/db/schema/public.ts b/apps/api/src/db/schema/public.ts
index 9af4190d5..e86c8ba23 100644
--- a/apps/api/src/db/schema/public.ts
+++ b/apps/api/src/db/schema/public.ts
@@ -96,6 +96,7 @@ export const batch_scrapes = pgTable("batch_scrapes", {
export const blocklist = pgTable("blocklist", {
id: bigintNum("id").notNull().generatedByDefaultAsIdentity(),
data: jsonb("data").notNull(),
+ org_id: uuid("org_id"),
});
export const blocklist_hits = pgTable("blocklist_hits", {
diff --git a/apps/api/src/lib/deep-research/deep-research-service.ts b/apps/api/src/lib/deep-research/deep-research-service.ts
index 16b36b300..ef0853e44 100644
--- a/apps/api/src/lib/deep-research/deep-research-service.ts
+++ b/apps/api/src/lib/deep-research/deep-research-service.ts
@@ -133,6 +133,7 @@ export async function performDeepResearch(options: DeepResearchServiceOptions) {
searchQuery.query,
{
teamId: options.teamId,
+ orgId: acuc?.org_id ?? null,
origin: "deep-research",
timeout: 10000,
scrapeOptions: {
diff --git a/apps/api/src/lib/engpicker.ts b/apps/api/src/lib/engpicker.ts
index 74fddfdb8..1e19e08c5 100644
--- a/apps/api/src/lib/engpicker.ts
+++ b/apps/api/src/lib/engpicker.ts
@@ -48,6 +48,7 @@ async function evaluateURL(
{
forceEngine: engine,
teamId: "engpicker",
+ orgId: null,
},
new CostTracking(),
);
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 1da8e2740..0246ab098 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -17,6 +17,7 @@ import { scrapeQueue } from "../../services/worker/nuq-router";
interface ScrapeDocumentOptions {
url: string;
teamId: string;
+ orgId?: string | null;
origin: string;
timeout: number;
isSingleUrl?: boolean;
@@ -40,6 +41,7 @@ export async function scrapeDocument(
if (
isUrlBlocked(options.url, options.flags ?? null, {
team_id: options.teamId,
+ org_id: options.orgId ?? null,
origin: options.origin,
})
) {
@@ -64,6 +66,7 @@ export async function scrapeDocument(
}),
internalOptions: {
teamId: options.teamId,
+ orgId: options.orgId ?? null,
saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME
? true
: false,
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index 08a0b0b77..7a4522099 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -17,6 +17,7 @@ import type { ThreatProtectionPolicy } from "../../threat-protection/types";
interface ScrapeDocumentOptions {
url: string;
teamId: string;
+ orgId?: string | null;
origin: string;
timeout: number;
isSingleUrl?: boolean;
@@ -42,6 +43,7 @@ export async function scrapeDocument_F0(
if (
isUrlBlocked(options.url, options.flags ?? null, {
team_id: options.teamId,
+ org_id: options.orgId ?? null,
origin: options.origin,
})
) {
@@ -68,6 +70,7 @@ export async function scrapeDocument_F0(
scrapeOptions,
internalOptions: {
teamId: options.teamId,
+ orgId: options.orgId ?? null,
bypassBilling: true,
threatProtection: options.threatProtectionPolicy ?? undefined,
},
diff --git a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
index 3613903ff..553be8a86 100644
--- a/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/extraction-service-f0.ts
@@ -180,6 +180,7 @@ export async function performExtraction_F0(
url,
prompt: request.prompt,
teamId,
+ orgId: acuc?.org_id ?? null,
allowExternalLinks: request.allowExternalLinks,
origin: request.origin,
limit: request.limit,
@@ -348,6 +349,7 @@ export async function performExtraction_F0(
{
url,
teamId,
+ orgId: acuc?.org_id ?? null,
origin: "extract",
timeout,
flags: acuc?.flags ?? null,
@@ -644,6 +646,7 @@ export async function performExtraction_F0(
{
url,
teamId,
+ orgId: acuc?.org_id ?? null,
origin: "extract",
timeout,
flags: acuc?.flags ?? null,
diff --git a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts
index 650288a46..c97b2f636 100644
--- a/apps/api/src/lib/extract/fire-0/url-processor-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/url-processor-f0.ts
@@ -53,6 +53,7 @@ interface ProcessUrlOptions {
prompt?: string;
schema?: any;
teamId: string;
+ orgId?: string | null;
allowExternalLinks?: boolean;
origin?: string;
limit?: number;
@@ -80,6 +81,7 @@ export async function processUrl_F0(
if (
!isUrlBlocked(options.url, teamFlags, {
team_id: options.teamId,
+ org_id: options.orgId ?? null,
origin: options.origin ?? null,
})
) {
@@ -117,6 +119,7 @@ export async function processUrl_F0(
url: baseUrl,
search: searchQuery,
teamId: options.teamId,
+ orgId: options.orgId ?? null,
allowExternalLinks: options.allowExternalLinks,
origin: options.origin,
limit: options.limit,
@@ -154,6 +157,7 @@ export async function processUrl_F0(
const retryMapResults = await getMapResults({
url: baseUrl,
teamId: options.teamId,
+ orgId: options.orgId ?? null,
allowExternalLinks: options.allowExternalLinks,
origin: options.origin,
limit: options.limit,
diff --git a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
index 71da9bb21..f841c2c82 100644
--- a/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
+++ b/apps/api/src/lib/generate-llmstxt/generate-llmstxt-service.ts
@@ -3,6 +3,7 @@ import { updateGeneratedLlmsTxt } from "./generate-llmstxt-redis";
import { getMapResults } from "../../controllers/v1/map";
import { z } from "zod";
import { scrapeDocument } from "../extract/document-scraper";
+import { hasOrgScopedBlocklist } from "../../scraper/WebScraper/utils/blocklist";
import {
getLlmsTextFromCache,
saveLlmsTextToCache,
@@ -87,10 +88,18 @@ export async function performGenerateLlmsTxt(
// Enforce max URL limit
const effectiveMaxUrls = Math.min(maxUrls, 5000);
+ // The cache is shared across teams, so a requester under org-scoped
+ // blocklist rules must not touch it at all: even with the origin URL
+ // itself unblocked, mapped child URLs may still be blocked, so a cached
+ // generation could hand them blocked content and their filtered output
+ // would poison the cache for everyone else.
+ const skipSharedCache = hasOrgScopedBlocklist(acuc?.org_id);
+
// Check cache first, unless cache is set to false
- const cachedResult = cache
- ? await getLlmsTextFromCache(url, effectiveMaxUrls)
- : null;
+ const cachedResult =
+ cache && !skipSharedCache
+ ? await getLlmsTextFromCache(url, effectiveMaxUrls)
+ : null;
if (cachedResult) {
logger.info("Found cached LLMs text", { url });
@@ -130,6 +139,7 @@ export async function performGenerateLlmsTxt(
const mapResult = await getMapResults({
url,
teamId,
+ orgId: acuc?.org_id ?? null,
limit: effectiveMaxUrls,
includeSubdomains: false,
ignoreSitemap: false,
@@ -159,6 +169,7 @@ export async function performGenerateLlmsTxt(
{
url,
teamId,
+ orgId: acuc?.org_id ?? null,
origin: "llmstxt",
timeout: 30000,
isSingleUrl: true,
@@ -233,7 +244,9 @@ export async function performGenerateLlmsTxt(
}
// After successful generation, save to cache
- await saveLlmsTextToCache(url, llmstxt, llmsFulltxt, effectiveMaxUrls);
+ if (!skipSharedCache) {
+ await saveLlmsTextToCache(url, llmstxt, llmsFulltxt, effectiveMaxUrls);
+ }
// Limit pages and remove separators before final update
const limitedFullText = limitPages(llmsFulltxt, effectiveMaxUrls);
diff --git a/apps/api/src/lib/map-utils.ts b/apps/api/src/lib/map-utils.ts
index 43bbc8b66..056d4a7cc 100644
--- a/apps/api/src/lib/map-utils.ts
+++ b/apps/api/src/lib/map-utils.ts
@@ -86,6 +86,7 @@ export async function getMapResults({
includeSubdomains = true,
crawlerOptions = {},
teamId,
+ orgId,
allowExternalLinks,
abort = new AbortController().signal,
filterByPath = true,
@@ -103,6 +104,7 @@ export async function getMapResults({
includeSubdomains?: boolean;
crawlerOptions?: any;
teamId: string;
+ orgId?: string | null;
origin?: string;
includeMetadata?: boolean;
allowExternalLinks?: boolean;
@@ -144,7 +146,7 @@ export async function getMapResults({
...(location ? { location } : {}),
...(headers ? { headers } : {}),
}),
- internalOptions: { teamId },
+ internalOptions: { teamId, orgId: orgId ?? null },
team_id: teamId,
createdAt: Date.now(),
zeroDataRetention,
@@ -370,6 +372,7 @@ export async function buildPromptWithWebsiteStructure({
basePrompt,
url,
teamId,
+ orgId,
flags,
logger,
limit = 50,
@@ -381,6 +384,7 @@ export async function buildPromptWithWebsiteStructure({
basePrompt: string;
url: string;
teamId: string;
+ orgId?: string | null;
flags: TeamFlags | null;
logger: Logger;
limit?: number;
@@ -397,6 +401,7 @@ export async function buildPromptWithWebsiteStructure({
includeSubdomains,
crawlerOptions: { sitemap: "include" },
teamId,
+ orgId: orgId ?? null,
flags,
allowExternalLinks,
filterByPath: false,
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
index 1b27a8c0a..4e8f5e00f 100644
--- a/apps/api/src/lib/robots-txt.ts
+++ b/apps/api/src/lib/robots-txt.ts
@@ -79,6 +79,7 @@ export async function fetchRobotsTxt(
{
forceEngine,
v0DisableJsDom: true,
+ orgId: null,
externalAbort: abort
? {
signal: abort,
diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts
index 9c1f9e768..fd9cc5a41 100644
--- a/apps/api/src/lib/scrape-billing.test.ts
+++ b/apps/api/src/lib/scrape-billing.test.ts
@@ -13,6 +13,7 @@ describe("calculateCreditsToBeBilled", () => {
} as any,
{
teamId: "team-id",
+ orgId: null,
},
{
metadata: {
@@ -40,6 +41,7 @@ describe("calculateCreditsToBeBilled", () => {
} as any,
{
teamId: "team-id",
+ orgId: null,
},
{
metadata: {
@@ -64,6 +66,7 @@ describe("calculateCreditsToBeBilled", () => {
} as any,
{
teamId: "team-id",
+ orgId: null,
},
{
metadata: {
@@ -95,6 +98,7 @@ describe("calculateCreditsToBeBilled", () => {
} as any,
{
teamId: "team-id",
+ orgId: null,
},
{
metadata: {
@@ -163,7 +167,7 @@ const billWithDecisions = (args: {
}) =>
calculateCreditsToBeBilled(
{ formats: [{ type: "markdown" }] } as any,
- { teamId: "team-id" },
+ { teamId: "team-id", orgId: null },
args.document as any,
{ totalCost: 0 } as any,
{} as any,
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index c80c7dd3b..c52d15f8f 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -29,6 +29,8 @@ export async function startWebScraperPipeline({
crawlId: job.data.crawl_id,
teamId: job.data.team_id,
...job.data.internalOptions,
+ // In-flight jobs enqueued before orgId existed may omit it.
+ orgId: job.data.internalOptions?.orgId ?? null,
},
team_id: job.data.team_id,
bull_job_id: job.id,
@@ -89,6 +91,7 @@ async function runWebScraper({
...internalOptions,
urlInvisibleInCurrentCrawl,
teamId: internalOptions?.teamId ?? team_id,
+ orgId: internalOptions?.orgId ?? null,
},
costTracking,
);
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 73f553e3b..d3e78aae9 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -341,6 +341,7 @@ function blocklistGate(
!canUseExchange &&
isUrlBlocked(req.body.url, req.acuc?.flags ?? null, {
team_id: req.acuc?.team_id ?? null,
+ org_id: req.acuc?.org_id ?? null,
origin: typeof req.body.origin === "string" ? req.body.origin : null,
})
) {
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 81824a3d0..62d240bed 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -115,6 +115,7 @@ export async function getLinksFromSitemap(
{
forceEngine,
v0DisableJsDom: true,
+ orgId: null,
externalAbort: abort
? {
signal: abort,
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts
new file mode 100644
index 000000000..c290111ac
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.test.ts
@@ -0,0 +1,223 @@
+import type { TeamFlags } from "../../../controllers/v1/types";
+import {
+ hasOrgScopedBlocklist,
+ initializeBlocklist,
+ isUrlBlocked,
+} from "./blocklist";
+
+const dbState = vi.hoisted(() => ({
+ rows: [] as { data: unknown; org_id: string | null }[],
+}));
+
+vi.mock("../../../config", () => ({
+ config: {
+ USE_DB_AUTHENTICATION: true,
+ DISABLE_BLOCKLIST: false,
+ },
+}));
+
+vi.mock("../../../db/connection", () => ({
+ db: {
+ insert: vi.fn(() => ({ values: vi.fn(async () => {}) })),
+ },
+ dbRr: {
+ select: vi.fn(() => ({
+ from: vi.fn(async () => dbState.rows),
+ })),
+ },
+}));
+
+const ORG_A = "00000000-0000-0000-0000-00000000000a";
+const ORG_B = "00000000-0000-0000-0000-00000000000b";
+const TEAM = "00000000-0000-0000-0000-000000000001";
+
+// NOTE: the matcher also blocks other TLDs of the same base domain, so
+// placeholder domains here must not share a base name across lists unless
+// the test is exercising exactly that rule.
+const GLOBAL_ROW = {
+ org_id: null,
+ data: {
+ blocklist: ["facebook.com"],
+ allowedKeywords: ["/legal"],
+ },
+};
+
+function ctx(org_id: string | null) {
+ return { team_id: TEAM, org_id, origin: "test" };
+}
+
+describe("isUrlBlocked", () => {
+ describe("global entries", () => {
+ beforeAll(async () => {
+ dbState.rows = [GLOBAL_ROW];
+ await initializeBlocklist();
+ });
+
+ it("blocks a global entry with no context", () => {
+ expect(isUrlBlocked("https://facebook.com/profile", null)).toBe(true);
+ });
+
+ it("blocks a global entry for any org", () => {
+ expect(
+ isUrlBlocked("https://facebook.com/profile", null, ctx(ORG_A)),
+ ).toBe(true);
+ expect(
+ isUrlBlocked("https://facebook.com/profile", null, ctx(null)),
+ ).toBe(true);
+ });
+
+ it("does not block unlisted domains", () => {
+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_A))).toBe(false);
+ });
+
+ it("respects global allowedKeywords", () => {
+ expect(isUrlBlocked("https://facebook.com/legal", null)).toBe(false);
+ });
+
+ it("unblocks a global entry via flags.unblockedDomains", () => {
+ const flags = { unblockedDomains: ["facebook.com"] } as TeamFlags;
+ expect(isUrlBlocked("https://facebook.com/profile", flags)).toBe(false);
+ });
+ });
+
+ describe("org-scoped entries", () => {
+ beforeAll(async () => {
+ dbState.rows = [
+ GLOBAL_ROW,
+ {
+ org_id: ORG_A,
+ data: {
+ blocklist: ["example.com", "org-blocked.com"],
+ allowedKeywords: ["/allowed-path"],
+ },
+ },
+ ];
+ await initializeBlocklist();
+ });
+
+ it("blocks the org's entry for that org only", () => {
+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_A))).toBe(true);
+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_B))).toBe(false);
+ expect(isUrlBlocked("https://example.com", null, ctx(null))).toBe(false);
+ expect(isUrlBlocked("https://example.com", null)).toBe(false);
+ });
+
+ it("keeps blocking global entries for the org", () => {
+ expect(
+ isUrlBlocked("https://facebook.com/profile", null, ctx(ORG_A)),
+ ).toBe(true);
+ });
+
+ it("blocks subdomains of an org-scoped entry", () => {
+ expect(
+ isUrlBlocked("https://sub.example.com/page", null, ctx(ORG_A)),
+ ).toBe(true);
+ });
+
+ it("blocks different TLDs of an org-scoped entry", () => {
+ expect(isUrlBlocked("https://example.de", null, ctx(ORG_A))).toBe(true);
+ });
+
+ it("unblocks org-scoped entries via flags.unblockedDomains", () => {
+ const flags = { unblockedDomains: ["example.com"] } as TeamFlags;
+ expect(isUrlBlocked("https://example.com", flags, ctx(ORG_A))).toBe(
+ false,
+ );
+ // Only the listed domain is unblocked, not the org's other entries.
+ expect(isUrlBlocked("https://org-blocked.com", flags, ctx(ORG_A))).toBe(
+ true,
+ );
+ });
+
+ it("respects the org blob's own allowedKeywords for its list only", () => {
+ expect(
+ isUrlBlocked("https://example.com/allowed-path", null, ctx(ORG_A)),
+ ).toBe(false);
+ // The org's allowedKeywords do not exempt globally-blocked URLs.
+ expect(
+ isUrlBlocked("https://facebook.com/allowed-path", null, ctx(ORG_A)),
+ ).toBe(true);
+ });
+
+ it("reports which orgs have org-scoped entries", () => {
+ expect(hasOrgScopedBlocklist(ORG_A)).toBe(true);
+ expect(hasOrgScopedBlocklist(ORG_B)).toBe(false);
+ expect(hasOrgScopedBlocklist(null)).toBe(false);
+ expect(hasOrgScopedBlocklist(undefined)).toBe(false);
+ });
+
+ it("does not count an org whose rows hold no blockable entries", async () => {
+ dbState.rows = [
+ GLOBAL_ROW,
+ { org_id: ORG_B, data: { blocklist: [], allowedKeywords: ["/x"] } },
+ ];
+ await initializeBlocklist();
+ expect(hasOrgScopedBlocklist(ORG_B)).toBe(false);
+ });
+
+ it("does not count an org whose entries are all blank strings", async () => {
+ dbState.rows = [
+ GLOBAL_ROW,
+ {
+ org_id: ORG_B,
+ data: { blocklist: ["", " "], allowedKeywords: [] },
+ },
+ ];
+ await initializeBlocklist();
+ expect(hasOrgScopedBlocklist(ORG_B)).toBe(false);
+ });
+ });
+
+ describe("row loading", () => {
+ it("merges multiple rows for the same org", async () => {
+ dbState.rows = [
+ GLOBAL_ROW,
+ {
+ org_id: ORG_A,
+ data: { blocklist: ["example-one.com"], allowedKeywords: [] },
+ },
+ {
+ org_id: ORG_A,
+ data: { blocklist: ["example-two.com"], allowedKeywords: [] },
+ },
+ ];
+ await initializeBlocklist();
+
+ expect(isUrlBlocked("https://example-one.com", null, ctx(ORG_A))).toBe(
+ true,
+ );
+ expect(isUrlBlocked("https://example-two.com", null, ctx(ORG_A))).toBe(
+ true,
+ );
+ });
+
+ it("tolerates malformed org row data without breaking loading", async () => {
+ dbState.rows = [
+ GLOBAL_ROW,
+ { org_id: ORG_A, data: null },
+ { org_id: ORG_B, data: { blocklist: "not-an-array" } },
+ ];
+ await initializeBlocklist();
+
+ expect(isUrlBlocked("https://facebook.com/profile", null)).toBe(true);
+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_A))).toBe(false);
+ expect(isUrlBlocked("https://example.com", null, ctx(ORG_B))).toBe(false);
+ });
+
+ it("throws when the table is empty", async () => {
+ dbState.rows = [];
+ await expect(initializeBlocklist()).rejects.toThrow(
+ "No data returned from database",
+ );
+ });
+
+ it("throws when rows exist but none is global", async () => {
+ dbState.rows = [
+ { org_id: ORG_A, data: { blocklist: [], allowedKeywords: [] } },
+ ];
+ await expect(initializeBlocklist()).rejects.toThrow(
+ "No global blocklist row",
+ );
+ });
+ });
+});
diff --git a/apps/api/src/scraper/WebScraper/utils/blocklist.ts b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
index eabf66ed1..04a758b55 100644
--- a/apps/api/src/scraper/WebScraper/utils/blocklist.ts
+++ b/apps/api/src/scraper/WebScraper/utils/blocklist.ts
@@ -15,6 +15,7 @@ type BlocklistBlob = {
type BlockContext = {
team_id?: string | null;
+ org_id?: string | null;
origin?: string | null;
};
@@ -102,6 +103,40 @@ function allowedKeywordMatches(url: string, allowedKeyword: string): boolean {
}
let blob: BlocklistBlob | null = null;
+let orgBlobs: Map<string, BlocklistBlob> = new Map();
+
+// Rows are written by ops directly, so reads must survive a missing or
+// partial document — a bad row must never take down startup. Blank entries
+// can never match and would make an org register as scoped, so drop them.
+function parseStringEntries(value: unknown): string[] {
+ return Array.isArray(value)
+ ? value.filter(
+ (x): x is string => typeof x === "string" && x.trim().length > 0,
+ )
+ : [];
+}
+
+function parseBlob(data: unknown): BlocklistBlob {
+ const doc = (data ?? {}) as Record<string, unknown>;
+ return {
+ blocklist: parseStringEntries(doc.blocklist),
+ allowedKeywords: parseStringEntries(doc.allowedKeywords),
+ };
+}
+
+function mergeBlob(target: BlocklistBlob, source: BlocklistBlob): void {
+ target.blocklist.push(...source.blocklist);
+ target.allowedKeywords.push(...source.allowedKeywords);
+}
+
+// Rows may repeat entries; the match loops scan linearly, so dedupe once at
+// load instead of paying for duplicates on every check.
+function dedupeBlob(blob: BlocklistBlob): BlocklistBlob {
+ return {
+ blocklist: [...new Set(blob.blocklist)],
+ allowedKeywords: [...new Set(blob.allowedKeywords)],
+ };
+}
export async function initializeBlocklist() {
if (config.USE_DB_AUTHENTICATION !== true || config.DISABLE_BLOCKLIST) {
@@ -109,79 +144,96 @@ export async function initializeBlocklist() {
blocklist: [],
allowedKeywords: [],
};
+ orgBlobs = new Map();
return;
}
- let data: { data: any } | undefined;
+ let rows: { data: any; org_id: string | null }[];
try {
- [data] = await dbRr.select().from(schema.blocklist).limit(1);
+ rows = await dbRr.select().from(schema.blocklist);
} catch (error) {
throw new Error(
`Error getting blocklist: ${error instanceof Error ? error.message : JSON.stringify(error)}`,
);
}
- if (!data) {
+ if (rows.length === 0) {
throw new Error("Error getting blocklist: No data returned from database");
}
- blob = data.data;
-}
-export function isUrlBlocked(
- url: string,
- flags: TeamFlags,
- context?: BlockContext,
-): boolean {
- if (blob === null) {
- throw new Error("Blocklist not initialized");
+ if (!rows.some(row => row.org_id === null)) {
+ throw new Error(
+ "Error getting blocklist: No global blocklist row (org_id IS NULL) found",
+ );
}
- const lowerCaseUrl = url.trim().toLowerCase();
-
- let blockedlist = [...blob.blocklist];
-
- if (flags?.unblockedDomains) {
- blockedlist = blockedlist.filter(
- blocked => !flags.unblockedDomains!.includes(blocked),
- );
+ const globalBlob: BlocklistBlob = { blocklist: [], allowedKeywords: [] };
+ const perOrg = new Map<string, BlocklistBlob>();
+ for (const row of rows) {
+ if (row.org_id === null) {
+ mergeBlob(globalBlob, parseBlob(row.data));
+ } else {
+ const existing = perOrg.get(row.org_id);
+ if (existing) {
+ mergeBlob(existing, parseBlob(row.data));
+ } else {
+ perOrg.set(row.org_id, parseBlob(row.data));
+ }
+ }
}
+ for (const [orgId, orgBlob] of perOrg) {
+ const deduped = dedupeBlob(orgBlob);
+ // An org whose rows hold no blockable entries must not register as
+ // org-scoped at all — hasOrgScopedBlocklist consumers would otherwise
+ // pay cache opt-outs for an org that can never block anything.
+ if (deduped.blocklist.length === 0) {
+ perOrg.delete(orgId);
+ } else {
+ perOrg.set(orgId, deduped);
+ }
+ }
+ blob = dedupeBlob(globalBlob);
+ orgBlobs = perOrg;
+}
+function findBlockedMatch(
+ url: string,
+ lowerCaseUrl: string,
+ blockedlist: string[],
+ allowedKeywords: string[],
+): string | null {
const decryptedUrl =
blockedlist.find(decrypted => lowerCaseUrl === decrypted) || lowerCaseUrl;
- // If the URL is empty or invalid, return false
+ // If the URL is empty or invalid, return no match
let parsedUrl: any;
try {
parsedUrl = parse(decryptedUrl);
} catch {
console.log("Error parsing URL:", url);
- return false;
+ return null;
}
const domain = parsedUrl.domain;
const publicSuffix = parsedUrl.publicSuffix;
if (!domain) {
- return false;
+ return null;
}
// Check if URL contains any allowed keyword
- if (
- blob.allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))
- ) {
- return false;
+ if (allowedKeywords.some(keyword => allowedKeywordMatches(url, keyword))) {
+ return null;
}
// Block exact matches
if (blockedlist.includes(domain)) {
- recordHit(url, domain, context);
- return true;
+ return domain;
}
// Block subdomains
if (blockedlist.some(blocked => domain.endsWith(`.${blocked}`))) {
- recordHit(url, domain, context);
- return true;
+ return domain;
}
// Block different TLDs of the same base domain
@@ -194,9 +246,76 @@ export function isUrlBlocked(
blocked => blocked.startsWith(baseDomain + ".") && blocked !== domain,
)
) {
- recordHit(url, domain, context);
+ return domain;
+ }
+
+ return null;
+}
+
+/**
+ * Whether the org has any org-scoped blocklist entries at all. Lets shared
+ * caches opt such requesters out entirely: a per-URL check is not enough
+ * when the cached artifact covers other URLs than the one checked.
+ */
+export function hasOrgScopedBlocklist(
+ orgId: string | null | undefined,
+): boolean {
+ if (blob === null) {
+ throw new Error("Blocklist not initialized");
+ }
+ return typeof orgId === "string" && orgBlobs.has(orgId);
+}
+
+export function isUrlBlocked(
+ url: string,
+ flags: TeamFlags,
+ context?: BlockContext,
+): boolean {
+ if (blob === null) {
+ throw new Error("Blocklist not initialized");
+ }
+
+ const lowerCaseUrl = url.trim().toLowerCase();
+
+ let blockedlist = [...blob.blocklist];
+
+ if (flags?.unblockedDomains) {
+ blockedlist = blockedlist.filter(
+ blocked => !flags.unblockedDomains!.includes(blocked),
+ );
+ }
+
+ const globalMatch = findBlockedMatch(
+ url,
+ lowerCaseUrl,
+ blockedlist,
+ blob.allowedKeywords,
+ );
+ if (globalMatch !== null) {
+ recordHit(url, globalMatch, context);
return true;
}
+ // The org blob's own allowedKeywords exempt URLs from the org list only.
+ const orgBlob = context?.org_id ? orgBlobs.get(context.org_id) : undefined;
+ if (orgBlob) {
+ let orgBlockedlist = orgBlob.blocklist;
+ if (flags?.unblockedDomains) {
+ orgBlockedlist = orgBlockedlist.filter(
+ blocked => !flags.unblockedDomains!.includes(blocked),
+ );
+ }
+ const orgMatch = findBlockedMatch(
+ url,
+ lowerCaseUrl,
+ orgBlockedlist,
+ orgBlob.allowedKeywords,
+ );
+ if (orgMatch !== null) {
+ recordHit(url, orgMatch, context);
+ return true;
+ }
+ }
+
return false;
}
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
index dfa68556e..040de8599 100644
--- a/apps/api/src/scraper/crawler/sitemap.ts
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -88,6 +88,7 @@ async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {
v0DisableJsDom: true,
// externalAbort: options.abort,
teamId: "sitemap",
+ orgId: null,
zeroDataRetention: options.zeroDataRetention,
crawlId: options.crawlId,
isPreCrawl: options.isPreCrawl,
diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.ts
index 60b986bc7..9660a4584 100644
--- a/apps/api/src/scraper/scrapeURL/engines/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts
@@ -7,10 +7,7 @@ import {
scrapeURLWithFireEngineChromeCDP,
scrapeURLWithFireEngineTLSClient,
} from "./fire-engine";
-import {
- exchangeMaxReasonableTime,
- scrapeURLWithExchange,
-} from "./exchange";
+import { exchangeMaxReasonableTime, scrapeURLWithExchange } from "./exchange";
import { pdfMaxReasonableTime, scrapePDF } from "./pdf";
import { fetchMaxReasonableTime, scrapeURLWithFetch } from "./fetch";
import {
@@ -178,7 +175,7 @@ export type EngineScrapeResult = {
const engineHandlers: {
[E in Engine]: (meta: Meta) => Promise<EngineScrapeResult>;
} = {
- "exchange": scrapeURLWithExchange,
+ exchange: scrapeURLWithExchange,
index: scrapeURLWithIndex,
"index;documents": scrapeURLWithIndex,
"fire-engine;chrome-cdp": scrapeURLWithFireEngineChromeCDP,
@@ -198,7 +195,7 @@ const engineHandlers: {
const engineMRTs: {
[E in Engine]: (meta: Meta) => number;
} = {
- "exchange": exchangeMaxReasonableTime,
+ exchange: exchangeMaxReasonableTime,
index: indexMaxReasonableTime,
"index;documents": indexMaxReasonableTime,
"fire-engine;chrome-cdp": meta =>
@@ -231,7 +228,7 @@ const engineOptions: {
quality: number;
};
} = {
- "exchange": {
+ exchange: {
features: {
actions: false,
waitFor: false,
@@ -632,6 +629,7 @@ export async function buildFallbackList(meta: Meta): Promise<
meta.internalOptions.teamFlags ?? null,
{
team_id: meta.internalOptions.teamId ?? null,
+ org_id: meta.internalOptions.orgId ?? null,
origin: null,
},
)
@@ -639,10 +637,9 @@ export async function buildFallbackList(meta: Meta): Promise<
return [];
}
} catch (error) {
- meta.logger.warn(
- "Exchange blocklist re-check failed; failing closed",
- { error },
- );
+ meta.logger.warn("Exchange blocklist re-check failed; failing closed", {
+ error,
+ });
return [];
}
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index 904bd11cf..70be2d8de 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -493,6 +493,12 @@ export type InternalOptions = {
bypassBilling?: boolean;
zeroDataRetention?: boolean;
teamFlags?: TeamFlags;
+ /** Team's org, snapshotted from the request ACUC at acceptance (same
+ * pattern as teamFlags). Rides the job payload so org-scoped blocklist
+ * checks work without re-fetching the chunk. Required so a payload
+ * builder cannot silently omit the org and skip org-scoped enforcement;
+ * pass null when the caller genuinely has no org (internal/system work). */
+ orgId: string | null;
/** Team's sold concurrency, snapshotted from the request ACUC at
* acceptance (same pattern as teamFlags). Rides the job payload so
* downstream engines (FirePDF async account context) never re-fetch. */
diff --git a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
index e542cc93f..c5d4a6d9a 100644
--- a/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
+++ b/apps/api/src/scraper/scrapeURL/scrapeURL.test.ts
@@ -10,7 +10,7 @@ import { Engine } from "./engines";
import { CostTracking } from "../../lib/cost-tracking";
// Mock parseMarkdown but delegate to real implementation for other tests
-vi.mock("../../lib/html-to-markdown", async (importOriginal) => {
+vi.mock("../../lib/html-to-markdown", async importOriginal => {
const actual =
await importOriginal<typeof import("../../lib/html-to-markdown")>();
return {
@@ -40,7 +40,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-basic",
"https://firecrawl-test-site.vercel.app",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -79,7 +79,7 @@ describe("Standalone scrapeURL tests", () => {
scrapeOptions.parse({
formats: ["markdown", "html"],
}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -104,7 +104,7 @@ describe("Standalone scrapeURL tests", () => {
scrapeOptions.parse({
onlyMainContent: false,
}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -128,7 +128,7 @@ describe("Standalone scrapeURL tests", () => {
onlyMainContent: false,
excludeTags: [".nav", "#footer", "strong"],
}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -149,7 +149,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-400",
"https://httpstat.us/400",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -168,7 +168,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-401",
"https://httpstat.us/401",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -187,7 +187,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-403",
"https://httpstat.us/403",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -206,7 +206,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-404",
"https://httpstat.us/404",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -225,7 +225,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-405",
"https://httpstat.us/405",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -244,7 +244,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-500",
"https://httpstat.us/500",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -263,7 +263,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-redirect",
"https://scrapethissite.com/",
scrapeOptions.parse({}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -296,7 +296,7 @@ describe("Standalone scrapeURL tests", () => {
scrapeOptions.parse({
formats: ["screenshot"],
}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -325,7 +325,7 @@ describe("Standalone scrapeURL tests", () => {
scrapeOptions.parse({
formats: ["screenshot@fullPage"],
}),
- { forceEngine, teamId: "test" },
+ { forceEngine, teamId: "test", orgId: null },
new CostTracking(),
);
@@ -354,7 +354,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-pdf",
"https://arxiv.org/pdf/astro-ph/9301001.pdf",
scrapeOptions.parse({}),
- { teamId: "test" },
+ { teamId: "test", orgId: null },
new CostTracking(),
);
@@ -374,7 +374,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-docx",
"https://nvca.org/wp-content/uploads/2019/06/NVCA-Model-Document-Stock-Purchase-Agreement.docx",
scrapeOptions.parse({}),
- { teamId: "test" },
+ { teamId: "test", orgId: null },
new CostTracking(),
);
@@ -396,7 +396,7 @@ describe("Standalone scrapeURL tests", () => {
"test:scrape-xlsx",
"https://download.microsoft.com/download/1/4/E/14EDED28-6C58-4055-A65C-23B4DA81C4DE/Financial%20Sample.xlsx",
scrapeOptions.parse({}),
- { teamId: "test" },
+ { teamId: "test", orgId: null },
new CostTracking(),
);
@@ -437,7 +437,7 @@ describe("Standalone scrapeURL tests", () => {
},
},
}),
- { teamId: "test" },
+ { teamId: "test", orgId: null },
new CostTracking(),
);
@@ -474,7 +474,7 @@ describe("Standalone scrapeURL tests", () => {
},
},
}),
- { teamId: "test" },
+ { teamId: "test", orgId: null },
new CostTracking(),
);
@@ -501,7 +501,7 @@ describe("Standalone scrapeURL tests", () => {
id,
url,
scrapeOptions.parse({}),
- { teamId: "test" },
+ { teamId: "test", orgId: null },
new CostTracking(),
);
@@ -545,7 +545,7 @@ describe("Standalone scrapeURL tests", () => {
scrapeOptions.parse({
formats: ["rawHtml"],
}),
- { teamId: "sitemap" },
+ { teamId: "sitemap", orgId: null },
new CostTracking(),
);
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 5cab96b50..15f5186ef 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -46,6 +46,7 @@ interface SearchOptions {
interface SearchContext {
teamId: string;
+ orgId?: string | null;
origin: string;
integration?: string | null;
apiKeyId: number | null;
@@ -79,6 +80,7 @@ export async function executeSearch(
const { query, limit, sources, categories, scrapeOptions } = options;
const {
teamId,
+ orgId,
origin,
apiKeyId,
flags,
@@ -212,12 +214,14 @@ export async function executeSearch(
if (shouldScrape && scrapeOptions) {
const itemsToScrape = getItemsToScrape(searchResponse, flags, {
team_id: teamId,
+ org_id: orgId ?? null,
origin,
});
if (itemsToScrape.length > 0) {
const scrapeOpts = {
teamId,
+ orgId: orgId ?? null,
origin,
timeout: options.timeout,
scrapeOptions,
diff --git a/apps/api/src/search/scrape.ts b/apps/api/src/search/scrape.ts
index d14f4a3cb..5ac8c353e 100644
--- a/apps/api/src/search/scrape.ts
+++ b/apps/api/src/search/scrape.ts
@@ -31,6 +31,7 @@ interface ScrapeItem {
interface ScrapeSearchOptions {
teamId: string;
+ orgId?: string | null;
origin: string;
timeout: number;
scrapeOptions: ScrapeOptions;
@@ -79,6 +80,7 @@ async function scrapeSearchResultDirect(
},
internalOptions: {
teamId: options.teamId,
+ orgId: options.orgId ?? null,
bypassBilling: options.bypassBilling ?? true,
zeroDataRetention,
teamFlags: flags,
@@ -147,7 +149,11 @@ async function scrapeSearchResultDirect(
export function getItemsToScrape(
searchResponse: SearchV2Response,
flags: TeamFlags,
- context?: { team_id?: string | null; origin?: string | null },
+ context?: {
+ team_id?: string | null;
+ org_id?: string | null;
+ origin?: string | null;
+ },
): ScrapeItem[] {
const items: ScrapeItem[] = [];
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index cd671fd62..4858328ab 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -503,6 +503,7 @@ const processPrecrawlJob = async (token: string, job: Job) => {
internalOptions: {
disableSmartWaitCache: true, // NOTE: smart wait disabled for crawls to ensure contentful scrape, speed does not matter
teamId,
+ orgId: null, // internal pre-crawl team, no org applies
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
zeroDataRetention: false,
isPreCrawl: true, // NOTE: must be added to internal options for indexing, if not it will be treated as a normal scrape in the index
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 3b5849909..c3cd12dfe 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -303,6 +303,9 @@ async function scrapeSearchMonitorPage(params: {
scrapeOptions,
internalOptions: {
teamId: params.teamId,
+ // Monitors do no in-pipeline blocklist enforcement (business rule);
+ // search results are filtered via isBlocked before scraping.
+ orgId: null,
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
bypassBilling: true,
zeroDataRetention: false,
@@ -535,6 +538,8 @@ async function enqueueMonitorScrapeTarget(params: {
scrapeOptions,
internalOptions: {
teamId: params.monitor.team_id,
+ // Monitors do no in-pipeline blocklist enforcement (business rule).
+ orgId: null,
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
bypassBilling: true,
zeroDataRetention: false,
@@ -603,6 +608,8 @@ async function enqueueMonitorCrawlTarget(params: {
internalOptions: {
disableSmartWaitCache: true,
teamId: params.monitor.team_id,
+ // Monitors do no in-pipeline blocklist enforcement (business rule).
+ orgId: null,
saveScrapeResultToGCS: !!config.GCS_FIRE_ENGINE_BUCKET_NAME,
zeroDataRetention: false,
bypassBilling: true,
@@ -778,6 +785,7 @@ async function runMonitorSearchTarget(params: {
isBlocked: url =>
isUrlBlocked(url, teamFlags, {
team_id: monitor.team_id,
+ org_id: acuc?.org_id ?? null,
origin: "monitor.search",
}),
goalVersion,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index ac40ba261..bfabe4ead 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -464,15 +464,13 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
await saveCrawl(job.data.crawl_id, sc);
}
+ const teamChunk = await getACUCTeam(job.data.team_id);
if (
- isUrlBlocked(
- doc.metadata.url,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
- {
- team_id: job.data.team_id,
- origin: job.data.origin,
- },
- )
+ isUrlBlocked(doc.metadata.url, teamChunk?.flags ?? null, {
+ team_id: job.data.team_id,
+ org_id: teamChunk?.org_id ?? null,
+ origin: job.data.origin,
+ })
) {
throw new CrawlDenialError(UNSUPPORTED_SITE_MESSAGE); // TODO: make this its own error type that is ignored by error tracking
}