Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): scope deterministic JSON extractors built from custom request contexts to the team (#5653)
tomsideguidecae8bd1dd990
diff --git a/apps/api/src/lib/deterministicJson/config.ts b/apps/api/src/lib/deterministicJson/config.tsindex e365a2e7f..a6662e196 100644--- a/apps/api/src/lib/deterministicJson/config.ts+++ b/apps/api/src/lib/deterministicJson/config.ts@@ -6,7 +6,7 @@ export const LIGHT_MODEL = config.EXTRACT_LIGHT_MODEL;export const CODE_SANDBOX_URL = config.CODE_SANDBOX_URL;// Bump to invalidate every cached extractor at once.-export const CACHE_VERSION = 1;+export const CACHE_VERSION = 2;export const MARKDOWN_BUDGET = 50_000;export const HTML_BUDGET = 40_000;diff --git a/apps/api/src/lib/deterministicJson/extract.ts b/apps/api/src/lib/deterministicJson/extract.tsindex 2af8bf578..0e63e0aaf 100644--- a/apps/api/src/lib/deterministicJson/extract.ts+++ b/apps/api/src/lib/deterministicJson/extract.ts@@ -22,6 +22,8 @@ interface ExtractArgs {costTracking: CostTracking;maxAskLlmCalls?: number;forceRegenerate?: boolean;+ /** Keeps extractors generated from a non-public page view out of the shared cache. */+ cacheScope?: string;}export async function extractDeterministicJson(@@ -35,11 +37,19 @@ export async function extractDeterministicJson(sandbox,jsonSchema,costTracking,+ cacheScope,} = args;const schemaJson = JSON.stringify(jsonSchema);const key = sha(- [CACHE_VERSION, CODEGEN_MODEL, url, schemaJson, prompt].join("\0"),+ [+ CACHE_VERSION,+ CODEGEN_MODEL,+ url,+ schemaJson,+ prompt,+ ...(cacheScope ? [cacheScope] : []),+ ].join("\0"),);const meta = { url, model: CODEGEN_MODEL, cacheVersion: CACHE_VERSION };diff --git a/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.tsindex 357609f62..b35317629 100644--- a/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts@@ -3,6 +3,7 @@ import { Meta } from "..";import { Document } from "../../../controllers/v2/types";import { hasFormatOfType } from "../../../lib/format-utils";import { isUsOnly } from "../../../lib/us-only";+import { hasCustomRequestContext } from "../lib/request-context";import { eq } from "drizzle-orm";import { db, dbRr } from "../../../db/connection";import * as schema from "../../../db/schema";@@ -197,6 +198,9 @@ export async function performDeterministicJson(cache,sandbox: sandboxRunner(CODE_SANDBOX_URL),costTracking: meta.costTracking,+ cacheScope: hasCustomRequestContext(meta.options)+ ? meta.internalOptions.teamId+ : undefined,});} catch (error) {meta.logger.error("Deterministic JSON extraction failed", { error });