Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(pdf): enforce the per-team cap inside both by-reference placement functions (#4418)

Abimael Martell47260e6a9647
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts
index 9afca6216..4648dd539 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/fireEngineHandoff.test.ts
@@ -71,10 +71,16 @@ vi.mock("../../../../../lib/gcs-jobs", async () => {
import {
largePdfLimitBytes,
rewritePdfInputForFirePdf,
+ uploadPdfInputForFirePdf,
} from "../fire-pdf/by-reference";
import { downloadFireEngineGcsFile } from "../../utils/downloadGcsFile";
import { config } from "../../../../../config";
-import { stat, readFile as readFileAsync } from "node:fs/promises";
+import {
+ stat,
+ readFile as readFileAsync,
+ unlink,
+ writeFile,
+} from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
@@ -295,3 +301,111 @@ describe("fire-engine GCS handoff (positive paths, mocked storage)", () => {
);
});
});
+
+describe("per-team cap enforcement at the placement functions", () => {
+ const ORIGINAL_BUCKET = config.FIRE_ENGINE_PDF_GCS_BUCKET;
+ const ORIGINAL_IDS = config.PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS;
+ const ORIGINAL_DEFAULT = config.PDF_BY_REFERENCE_MAX_BYTES_DEFAULT;
+ const ORIGINAL_PRIVILEGED = config.PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED;
+ // Explicit caps, so the size assertions below can never drift from the
+ // schema defaults or be skewed by env overrides.
+ const DEFAULT_CAP = 50 * 1024 * 1024;
+ const PRIVILEGED_CAP = 200 * 1024 * 1024;
+ beforeAll(() => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = "fire-engine-scrape-storage";
+ });
+ afterAll(() => {
+ (config as any).FIRE_ENGINE_PDF_GCS_BUCKET = ORIGINAL_BUCKET;
+ });
+ beforeEach(() => {
+ gcsFake.reset();
+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = "team-privileged";
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_DEFAULT = DEFAULT_CAP;
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = PRIVILEGED_CAP;
+ });
+ afterEach(() => {
+ (config as any).PDF_BY_REFERENCE_PRIVILEGED_TEAM_IDS = ORIGINAL_IDS;
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_DEFAULT = ORIGINAL_DEFAULT;
+ (config as any).PDF_BY_REFERENCE_MAX_BYTES_PRIVILEGED = ORIGINAL_PRIVILEGED;
+ });
+
+ const HANDOFF = {
+ uri: "gs://fire-engine-scrape-storage/pdf-handoff/x.pdf",
+ sha256: "ab".repeat(32),
+ generation: "123456789012345678",
+ };
+
+ function metaForTeam(teamId: string) {
+ const meta = makeMeta();
+ meta.internalOptions.teamId = teamId;
+ return meta;
+ }
+
+ // Over the default tier, under the privileged one β€” the size that tells
+ // the two caps apart.
+ const OVER_DEFAULT_BYTES = DEFAULT_CAP + 1024 * 1024;
+
+ it("rewrite refuses a handoff above the default cap for an unlisted team", async () => {
+ await expect(
+ rewritePdfInputForFirePdf(metaForTeam("team-x"), {
+ ...HANDOFF,
+ sizeBytes: OVER_DEFAULT_BYTES,
+ }),
+ ).resolves.toBeNull();
+ expect(gcsFake.copies).toEqual([]);
+ });
+
+ it("rewrite allows the same size for a privileged team", async () => {
+ const res = await rewritePdfInputForFirePdf(
+ metaForTeam("team-privileged"),
+ {
+ ...HANDOFF,
+ sizeBytes: OVER_DEFAULT_BYTES,
+ },
+ );
+ expect(res).toMatchObject({ sizeBytes: OVER_DEFAULT_BYTES });
+ expect(gcsFake.copies).toHaveLength(1);
+ });
+
+ it("rewrite refuses above the privileged cap even for a privileged team", async () => {
+ await expect(
+ rewritePdfInputForFirePdf(metaForTeam("team-privileged"), {
+ ...HANDOFF,
+ sizeBytes: PRIVILEGED_CAP + 1024 * 1024,
+ }),
+ ).resolves.toBeNull();
+ expect(gcsFake.copies).toEqual([]);
+ });
+
+ it("upload refuses an over-cap file before reading a byte", async () => {
+ // The path deliberately does not exist: an over-cap upload must be
+ // refused by the size guard alone, before any disk or network I/O.
+ await expect(
+ uploadPdfInputForFirePdf(
+ metaForTeam("team-x"),
+ path.join(tmpdir(), "does-not-exist.pdf"),
+ OVER_DEFAULT_BYTES,
+ ),
+ ).resolves.toBeNull();
+ });
+
+ it("upload accepts a within-cap size for a privileged team", async () => {
+ const filePath = path.join(
+ tmpdir(),
+ `upload-test-${crypto.randomUUID()}.pdf`,
+ );
+ await writeFile(filePath, Buffer.alloc(1024, 7));
+ try {
+ const res = await uploadPdfInputForFirePdf(
+ metaForTeam("team-privileged"),
+ filePath,
+ // Declared size sits over the default tier but under the privileged
+ // one β€” the guard must use the team's own cap.
+ OVER_DEFAULT_BYTES,
+ );
+ expect(res).toMatchObject({ sizeBytes: OVER_DEFAULT_BYTES });
+ } finally {
+ await unlink(filePath);
+ }
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
index 5818ca67b..d2e3cecd3 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
@@ -71,6 +71,24 @@ export async function rewritePdfInputForFirePdf(
// bucket named by response data.
return null;
}
+ const limitBytes = largePdfLimitBytes(meta);
+ if (source.sizeBytes > limitBytes) {
+ // The routing gate already rejects oversized files, but this placement
+ // is the last step before bytes land in the fire-pdf bucket β€” it must
+ // enforce the team cap itself rather than trust every caller.
+ meta.logger.warn(
+ "fire-engine PDF handoff exceeds this team's large-PDF limit; refusing rewrite",
+ {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_by_reference_rewrite_over_cap",
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ size_bytes: source.sizeBytes,
+ limit_bytes: limitBytes,
+ },
+ );
+ return null;
+ }
const destBucket = config.FIRE_PDF_GCS_INPUT_BUCKET;
const destKey = firePdfInputObjectKey(meta.id);
const startedAt = Date.now();
@@ -151,7 +169,10 @@ export async function rewritePdfInputForFirePdf(
* architectural ceiling. Every acquisition path enforces this one number β€”
* the direct-download admission, the fire-engine handoff download, the
* by-reference routing gate, and (as pdfMaxSize) fire-engine's own capture
- * ceiling, so no path can admit bytes another would reject.
+ * ceiling, so no path can admit bytes another would reject. Both placement
+ * functions (the handoff rewrite and the streaming upload) enforce it once
+ * more themselves: nothing lands in the fire-pdf bucket above the team cap
+ * even if a caller skips the gates.
*/
export function largePdfLimitBytes(meta: Meta): number {
const teamId = meta.internalOptions.teamId;
@@ -269,6 +290,24 @@ export async function uploadPdfInputForFirePdf(
keyVariant?: string;
},
): Promise<FirePdfByReferenceInput | null> {
+ const limitBytes = largePdfLimitBytes(meta);
+ if (sizeBytes > limitBytes) {
+ // Same placement-level enforcement as the rewrite: the router's gate
+ // already rejects oversized files, but nothing may reach the fire-pdf
+ // bucket above the team cap regardless of which caller asked.
+ meta.logger.warn(
+ "Large PDF exceeds this team's large-PDF limit; refusing upload",
+ {
+ method: "scrapePDF/firePdfByReference",
+ event: "fire_pdf_by_reference_upload_over_cap",
+ scrape_id: meta.id,
+ team_id: meta.internalOptions.teamId,
+ size_bytes: sizeBytes,
+ limit_bytes: limitBytes,
+ },
+ );
+ return null;
+ }
const bucketName = config.FIRE_PDF_GCS_INPUT_BUCKET;
const objectKey = firePdfInputObjectKey(meta.id, opts?.keyVariant);
const startedAt = Date.now();