Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(pdf): enforce the per-team cap inside both by-reference placement functions (#4418)
Abimael Martell47260e6a9647
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.tsindex 9afca6216..4648dd539 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts@@ -71,10 +71,16 @@ vi.mock("../../../../../lib/gcs-jobs", async () => {import {largePdfLimitBytes,rewritePdfInputForFirePdf,+ uploadPdfInputForFirePdf,} from "../fire-pdf/by-reference";import { downloadFireEngineGcsFile } from "../../utils/downloadGcsFile";import { config } from "../../../../../config";-import { stat, readFile as readFileAsync } from "node:fs/promises";+import {+ stat,+ readFile as readFileAsync,+ unlink,+ writeFile,+} from "node:fs/promises";import { tmpdir } from "node:os";import path from "node:path";@@ -295,3 +301,111 @@ describe("fire-engine GCS handoff (positive paths, mocked storage)", () => {);});});++describe("per-team cap enforcement at the placement functions", () => {+ const ORIGINAL_BUCKET = config.FIRE_ENGINE_PDF_GCS_BUCKET;+ const ORIGINAL_IDS = config.PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS;+ const ORIGINAL_DEFAULT = config.PDF_BY_REFERENCE_MAX_BYTES_DEFAULT;+ const ORIGINAL_PRIVILEGED = config.PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED;+ // Explicit caps, so the size assertions below can never drift from the+ // schema defaults or be skewed by env overrides.+ const DEFAULT_CAP = 50 * 1024 * 1024;+ const PRIVILEGED_CAP = 200 * 1024 * 1024;+ beforeAll(() => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";+ });+ afterAll(() => {+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = ORIGINAL_BUCKET;+ });+ beforeEach(() => {+ gcsFake.reset();+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-privileged";+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_DEFAULT = DEFAULT_CAP;+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = PRIVILEGED_CAP;+ });+ afterEach(() => {+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = ORIGINAL_IDS;+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_DEFAULT = ORIGINAL_DEFAULT;+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = ORIGINAL_PRIVILEGED;+ });++ const HANDOFF = {+ uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf",+ sha256: "ab".repeat(32),+ generation: "123456789012345678",+ };++ function metaForTeam(teamId: string) {+ const meta = makeMeta();+ meta.internalOptions.teamId = teamId;+ return meta;+ }++ // Over the default tier, under the privileged one β the size that tells+ // the two caps apart.+ const OVER_DEFAULT_BYTES = DEFAULT_CAP + 1024 * 1024;++ it("rewrite refuses a handoff above the default cap for an unlisted team", async () => {+ await expect(+ rewritePdfInputForFirePdf(metaForTeam("team-x"), {+ ...HANDOFF,+ sizeBytes: OVER_DEFAULT_BYTES,+ }),+ ).resolves.toBeNull();+ expect(gcsFake.copies).toEqual([]);+ });++ it("rewrite allows the same size for a privileged team", async () => {+ const res = await rewritePdfInputForFirePdf(+ metaForTeam("team-privileged"),+ {+ ...HANDOFF,+ sizeBytes: OVER_DEFAULT_BYTES,+ },+ );+ expect(res).toMatchObject({ sizeBytes: OVER_DEFAULT_BYTES });+ expect(gcsFake.copies).toHaveLength(1);+ });++ it("rewrite refuses above the privileged cap even for a privileged team", async () => {+ await expect(+ rewritePdfInputForFirePdf(metaForTeam("team-privileged"), {+ ...HANDOFF,+ sizeBytes: PRIVILEGED_CAP + 1024 * 1024,+ }),+ ).resolves.toBeNull();+ expect(gcsFake.copies).toEqual([]);+ });++ it("upload refuses an over-cap file before reading a byte", async () => {+ // The path deliberately does not exist: an over-cap upload must be+ // refused by the size guard alone, before any disk or network I/O.+ await expect(+ uploadPdfInputForFirePdf(+ metaForTeam("team-x"),+ path.join(tmpdir(), "does-not-exist.pdf"),+ OVER_DEFAULT_BYTES,+ ),+ ).resolves.toBeNull();+ });++ it("upload accepts a within-cap size for a privileged team", async () => {+ const filePath = path.join(+ tmpdir(),+ `upload-test-${crypto.randomUUID()}.pdf`,+ );+ await writeFile(filePath, Buffer.alloc(1024, 7));+ try {+ const res = await uploadPdfInputForFirePdf(+ metaForTeam("team-privileged"),+ filePath,+ // Declared size sits over the default tier but under the privileged+ // one β the guard must use the team's own cap.+ OVER_DEFAULT_BYTES,+ );+ expect(res).toMatchObject({ sizeBytes: OVER_DEFAULT_BYTES });+ } finally {+ await unlink(filePath);+ }+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.tsindex 5818ca67b..d2e3cecd3 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts@@ -71,6 +71,24 @@ export async function rewritePdfInputForFirePdf(// bucket named by response data.return null;}+ const limitBytes = largePdfLimitBytes(meta);+ if (source.sizeBytes > limitBytes) {+ // The routing gate already rejects oversized files, but this placement+ // is the last step before bytes land in the fire-pdf bucket β it must+ // enforce the team cap itself rather than trust every caller.+ meta.logger.warn(+ "fire-engine PDF handoff exceeds this team's large-PDF limit; refusing rewrite",+ {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_by_reference_rewrite_over_cap",+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ size_bytes: source.sizeBytes,+ limit_bytes: limitBytes,+ },+ );+ return null;+ }const destBucket = config.FIRE_PDF_GCS_INPUT_BUCKET;const destKey = firePdfInputObjectKey(meta.id);const startedAt = Date.now();@@ -151,7 +169,10 @@ export async function rewritePdfInputForFirePdf(* architectural ceiling. Every acquisition path enforces this one number β* the direct-download admission, the fire-engine handoff download, the* by-reference routing gate, and (as pdfMaxSize) fire-engine's own capture- * ceiling, so no path can admit bytes another would reject.+ * ceiling, so no path can admit bytes another would reject. Both placement+ * functions (the handoff rewrite and the streaming upload) enforce it once+ * more themselves: nothing lands in the fire-pdf bucket above the team cap+ * even if a caller skips the gates.*/export function largePdfLimitBytes(meta: Meta): number {const teamId = meta.internalOptions.teamId;@@ -269,6 +290,24 @@ export async function uploadPdfInputForFirePdf(keyVariant?: string;},): Promise<FirePdfByReferenceInput | null> {+ const limitBytes = largePdfLimitBytes(meta);+ if (sizeBytes > limitBytes) {+ // Same placement-level enforcement as the rewrite: the router's gate+ // already rejects oversized files, but nothing may reach the fire-pdf+ // bucket above the team cap regardless of which caller asked.+ meta.logger.warn(+ "Large PDF exceeds this team's large-PDF limit; refusing upload",+ {+ method: "scrapePDF/firePdfByReference",+ event: "fire_pdf_by_reference_upload_over_cap",+ scrape_id: meta.id,+ team_id: meta.internalOptions.teamId,+ size_bytes: sizeBytes,+ limit_bytes: limitBytes,+ },+ );+ return null;+ }const bucketName = config.FIRE_PDF_GCS_INPUT_BUCKET;const objectKey = firePdfInputObjectKey(meta.id, opts?.keyVariant);const startedAt = Date.now();