Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): scope deterministic JSON extractors built from custom request contexts to the team (#5653)

tomsideguidecae8bd1dd990
diff --git a/apps/api/src/lib/deterministicJson/config.ts b/apps/api/src/lib/deterministicJson/config.ts
index e365a2e7f..a6662e196 100644
--- a/apps/api/src/lib/deterministicJson/config.ts
+++ b/apps/api/src/lib/deterministicJson/config.ts
@@ -6,7 +6,7 @@ export const LIGHT_MODEL = config.EXTRACT_LIGHT_MODEL;
export const CODE_SANDBOX_URL = config.CODE_SANDBOX_URL;
// Bump to invalidate every cached extractor at once.
-export const CACHE_VERSION = 1;
+export const CACHE_VERSION = 2;
export const MARKDOWN_BUDGET = 50_000;
export const HTML_BUDGET = 40_000;
diff --git a/apps/api/src/lib/deterministicJson/extract.ts b/apps/api/src/lib/deterministicJson/extract.ts
index 2af8bf578..0e63e0aaf 100644
--- a/apps/api/src/lib/deterministicJson/extract.ts
+++ b/apps/api/src/lib/deterministicJson/extract.ts
@@ -22,6 +22,8 @@ interface ExtractArgs {
costTracking: CostTracking;
maxAskLlmCalls?: number;
forceRegenerate?: boolean;
+ /** Keeps extractors generated from a non-public page view out of the shared cache. */
+ cacheScope?: string;
}
export async function extractDeterministicJson(
@@ -35,11 +37,19 @@ export async function extractDeterministicJson(
sandbox,
jsonSchema,
costTracking,
+ cacheScope,
} = args;
const schemaJson = JSON.stringify(jsonSchema);
const key = sha(
- [CACHE_VERSION, CODEGEN_MODEL, url, schemaJson, prompt].join("\0"),
+ [
+ CACHE_VERSION,
+ CODEGEN_MODEL,
+ url,
+ schemaJson,
+ prompt,
+ ...(cacheScope ? [cacheScope] : []),
+ ].join("\0"),
);
const meta = { url, model: CODEGEN_MODEL, cacheVersion: CACHE_VERSION };
diff --git a/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts
index 357609f62..b35317629 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/deterministicJson.ts
@@ -3,6 +3,7 @@ import { Meta } from "..";
import { Document } from "../../../controllers/v2/types";
import { hasFormatOfType } from "../../../lib/format-utils";
import { isUsOnly } from "../../../lib/us-only";
+import { hasCustomRequestContext } from "../lib/request-context";
import { eq } from "drizzle-orm";
import { db, dbRr } from "../../../db/connection";
import * as schema from "../../../db/schema";
@@ -197,6 +198,9 @@ export async function performDeterministicJson(
cache,
sandbox: sandboxRunner(CODE_SANDBOX_URL),
costTracking: meta.costTracking,
+ cacheScope: hasCustomRequestContext(meta.options)
+ ? meta.internalOptions.teamId
+ : undefined,
});
} catch (error) {
meta.logger.error("Deterministic JSON extraction failed", { error });