Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): send /v2/parse PDFs to FirePDF with dynamic table format (#5110)
Abimael Martell8604989d7526
diff --git a/apps/api/src/__tests__/snips/fixtures/parse-scanned.pdf b/apps/api/src/__tests__/snips/fixtures/parse-scanned.pdfnew file mode 100644index 000000000..f146cde68--- /dev/null+++ b/apps/api/src/__tests__/snips/fixtures/parse-scanned.pdf@@ -0,0 +1,79 @@+%PDF-1.3+%���� ReportLab Generated PDF document (opensource)+1 0 obj+<<+/F1 2 0 R+>>+endobj+2 0 obj+<<+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font+>>+endobj+3 0 obj+<<+/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 300 /Length 4581 /Subtype /Image+ /Type /XObject /Width 850+>>+stream+Gb"0SdB`&IXp%Z$s/3C<?+9*888MF,$t5.cfLqNCA7DLocR/\mzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!5NG6Nr8Y/595`ZQ_UYVkmGG\*#SbF%Y*i9r;u%dg4/ek(u0fi"5lWG-_IT'4)4fGf+j6tV,M>D!;s%AO#fN'DU#B!?fG-/!,5(j$%.B\;:emaTDdXin\R=B!1qjs"lCHsiiE*G3(a0kr#YV@SFY!FPQ7LbqBb\lZ>FlkkJpf74Qa/kB7DC9nT9c>DJ?EJoIiAMY33C!@H3C*p0h&tF]&-)G2V+\oVN_eK^#NgG;$WD<uO!*>'eVCPJd<4dm!/ORZ;&oQq3fVT>[Knl3F@J4PEG>Z!0V-a*P2?:@&qj"1N?j4$L>O#29=5:=HpV<Z3m)C4F[io$qQm;2kNYI'd::D./<IEinW=7oH`Wf1T8O>K#fe9@(5WH`km(o@Lk^2q@nZ:G7^c?ZXHs`R?MnL5i<e`PGT-ZM2S*@):KX4M<Nm(BhWLVfW]O,u:eriT!V(Z2SG$&PX?a3iK%.*FJfhZnf.0ACDLZT1/nMX7le$GDiQFh5HTVNNF,B'$6&%]Q6pB^kjb1;$Y;0a0E"[pFNbDIaIB5[8/$7m<8;XgAuN*FK<@W[-'R,W$TZtZE_qQc+)<6>Ap=1D1ej$I_87-Gr?\rH`H#1Ur,%OM2s'JXZhD[pi5h%bu2D3ae.=0:^5_#qiqf/):&(`)Yb2^GiO)7GNfjE`l3d*oqU1uW0Y60;0V]4\&rQ-<a9GUWU)\E\[email protected]*Uql@[@Y,VDVQK%7Gr`'gkcB64:Y4!AAENQlOX>_?Xs7/1[$l%nbk/*>>;r.&:pA:+?^74B+,971hfVW.f6RQn(N.N$cP*r=2n+*9;X)YQ3Zf5=/p#X#\rJ>nC8q6CGat]+aGR*OE][;O)a=b3h*.":jd:R&q!U(iS2aa$HU"%f*XT0;Vt&0oXV0X*XJQ@$aNW,TQduY(f+7/Ve[AB^(9ckgf!e>"aV?mt/)s/"mc4+Ea1b^fr/BkLep'F/n'[Od9:K8ACA$!"*]*B^Q28Fo"UfA,`m\T-F&p&):o>Jk:Ee/!:moB\Y.sUNMG;eP[d)W<:]%6]$iHC^4dNTRK"D`#e"qDsIT=Bl<M":];PLo)FBt$g1T?$Fe5$1bj/ke^oXT9PX`iG`MO&)-iu\)tr6midHJnOfh#f8'_t9O*g90P/ltt9qMgjsbhDt26il$U_g4,3^CNPQ<EmN:!B3]r6l)4.6G3kc%HEZtl4&6jkH"Zk@bC,bff`m_$2elEp`c$j55$e'RS9\XRnC1L:e<2W%`MH:\,?eh^n$_Tcfpf,<_iRM+Y<4B(4`TEC7BXRTo1FsL6%&"<<n"efV'B9WDss?b_@/Vfob"Tj/eP(Zc^k_K\j_JiH.'[lZE=_U;P^/K/'b*TH%A%lr">t[NtaNr]:r<q,#*Fm1iAa73)!>uF4d;S'@etRY#\19)F'51im"BngTd-Jn\^G]'sF_%msS<rNX#5CrJCk5J,HA"mCDuWXQq'ipM%:OGKtRbcXE$?&R@-,QD^l%pO7uXT71[u@V+STTeTQ[ZM2:ggKAit+9,O@lVd=HN7Dgpil5$8n.E@oC>#t<:rPPX2sE#U".\f0hP'8VVNkM2qLcrMP;[k,5+]M>*cN@7Mu'N(ol_6)&&sE)NakJ7)d>JW^GmFd%dm?0^phS$pXJ26duB5T[bC"_3(lj.Nu#spFo$*;F,3n(nR8AoMH;^-3,gs9)k_GP7/h:1E#9g_jK.:Do@9tgSn2(=kqh0?p^!#7l%6ftX+,OUI;YC+'.FE7Mt`Vl*s@V2Rc]Hf.$lD]*9hN5Ac)\OLDW%ocfQ>IWH^\D[s<JDV]'pEeijn"'eBO&[+4H*:0Vk#.qm\1lGFK5@&Zq?W^No+o(\_Z_S#U\QN^=\2JIYups@;iI5$.*3H/d,lRU@=^%T-Og1Ml%eZI,SF0h>/=j8&D-s^eZH2Q:dE8G@[6XC,NdOMqR4G8%,BnrOu,+TZsCs4f/=JX%tkFZVpr.+J[+PgeDE8#8Fa)Q*/X7;AFbf@n2Nq0`)[email protected]#-B?mcCT5mR&0Xes%8KOCl)Qa&&U?e:S0THHjS\>!!%M%?[VnSIt:sT!!#70^[M:)r"4DN!<E$[s/uq5i/dB.%fkN@n>d'"zzzzzzzzzzzzzzzzzzzzzzzzz!!!"L]6jIj:BuFTO8nb7/-,T.O+5]L>SjTQo*7Fa4rp.CV[:.O'u,SXr?D-\Y=QG)r[athIQ.5Hh)%$;5JR%Qo;h?+Egl?=duW9YA&Blt^5Vd`lMI(>rZ?Xfhr?QHn#"^X9k2'mUu/AV?d(b6f":[*N!!Y+RssF9)B3SQnZU`b97pNtr6(oIKs0T'g$GRX\XUTn)bI\ROX6&5Q_nLpN8:(6_\TeB\d$^[3Vi$JN&t9jVC`$meh+W#rlMW[BG]!9FoGjrHG!L<ml=#88VoBPYiP:2:YZnOrq3Wkab/?uoksWSWq_P%.HSi$dAeKabHjSt/2`CQ(#QUZD<Z=ucT#bsmZUtkPV_05S19YNROo@<[nAkj@eY1b+DCG%3id1HD$Q>3BeN(be@X=*\)IKJ:8umG--<SBmg"t2kZLSCkk;!L%@VAj+!`rg;9JsOAf-Jtk[0C;lVQIT#[2PJ#0#UL95e]YQf.0o/Y31$H/Z.!'tH5T8(2Up`IYLg=%e`sg&'F"h52A*j4aaL`B-T`/N.<JI[2Zo]K3iSreSHkOf0l%dN'gBJL*rBe=a=]r3.aBf;7k8o3?V)>]I:XHUc/hcF<S!p%`\MH=F+t@r4?QGhIF`V<kOd^N:%#",u^*rb5N;YJdIhDg?M6/:PnZDktpAjjLM;gt3A/><LbI^&,1YT$L3:8S,N:%q/`3BeFOTDBISIk<tNnG3K%Y[glU/:lk@5fK_dmD;Fn`a7%d*?Mj3MrEJ?SZ":t%T(_9H*<sK>%q4L7>>@!K#aLK9F8As[9,I'SnZ6tbC`k>#Y&/NblNNE1<hJL%]BH!$m^elPh5%R"k2g7K97=A]\+_`>h7A?,BQ__nV#g#K<idrtk_ci02NZMb*DtVUflZ,3CD)\Ll#kK5hFc+[%.USMG%k3q\tmbe_dHR3j5%;$pAFRIO0ahT>Q&?6GVs-I6W^qs!S,L4#b&JFnb;RgJk*V!@?q65`Y2Hs?1Xo$n`%]UMSlMH&&t2)cKai^LTU3:(?\ktLN_5uo&*S#Y88j'BUn&HG!"a7_cNEJ=>G>*,;4*!\iB%t?/!i-Ok=C"2.(4T41+6)\_3jHYNA(8>R;CLTCuuuh`S8:_ki,\/FP19[9>G)T$Dg_]]-9)4?F*/GhjQ^@`sc$dOLSAdNO`2P!%p%cDtf7,AiiHEPf9JM+9Pl):+_ODeUu0O*Bn.g/?7+Ur_9[(=UENg.:/E1iiCFNE:D"i_or!K=>mt_S3e7Oig(TQF+Js4YBI7e2]M:`kfBu]l\POW)rl'n(Y-^YZk3(ET[e/^\TKe"'jZkbK&XKI6^'RiCau4'B=#s;R]o3@$iUX?7rqT<h#]rpR^hq@A6,.H.T!9;_@FlFdeH-VDlcQI/8ufe)JAJ=88e?nakR$./:sq?cgB[d!NY5]:H?e/H<dS]K+)Kk<XObq8PKARLGL+NOj,6TA"Cd>KbBFXulTKJp=?\7SCQA`@\gqLuS`IF&qds]t\[<f._E^%#X'ShocXJ5BskHSY>.Yg<]NY-A1Dc"4)D>[VZp3hL)2??ZQ"&"'jZnDKWR^n]#qCS.b==HR0oG62\`Ir+fKERPl0u]$WsLcHnG.hXbTC]t1jli#hc^okk[DFnMZ>GIq^ajJ@s-BDDDaGV'.'FEr-XK==/,>oN0T)"28/](,2i"F.Qc(<?G3hQ\APqsqu<YF%oC1fs:`Dk29+Y\uN"kp*BdgcWL;(E[Yc@"Go0M;XPAgIeqIg9\2LqFKA6%JO(G/G4VQ?E<8;]DO?qUt`9&1eGkVGFdkh=^p\fIIKJ@K=.?NZpV&KOt,>^J*V`XVq^Tq\*\]r?B6=(9-OhWY1(q`e)dW+QaYk%Rnb:i[rG.?#T>Wo/73_gi`P1rM46D.g28cGI;"mkesAnL%d"__mVJZ_eqk:,p=^055-\Ki0K)`D]_BE=%\#k40"ppj)oY7/k;ii,mHQ^gUA/iHjl:6@-J'hfD\ZLr:DF9Xh`BCAOWh",B9t:ER>r]*f6'kdhlAnHf;#/Y!bo1]/b6>>g9,=!^DY&5YIJQFggfeW<1Y]tS8lI)SBo%dQfkq[qRdM1;FNL$DnXY;.77QX_6k(2O1,JA-b`mS5uuaCDU]^\\8_:XQ*ikPj[GZQVpi)Tq_hagHg[;=pXnF?`SKkCHe>-7il+L?>E,!$'>F)VCgHCn=MFh1MTZ6QcMfk2^:NbU5c*5`/s2.AnQHF$V$XrpiMuf%:(mUEn2oTq3)'D2N8FrV`+]8QL]!Y"06*E\^jt;K^Vjgf!,)4)pa&ZK!''&s"Fl9W"98Z(:A?,1'*&"4zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!!!#oT`5+S_eP&~>endstream+endobj+4 0 obj+<<+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject <<+/FormXob.c71196c43e59bbc614ba2df60682674c 3 0 R+>>+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+5 0 obj+<<+/PageMode /UseNone /Pages 7 0 R /Type /Catalog+>>+endobj+6 0 obj+<<+/Author (anonymous) /CreationDate (D:20000101000000+00'00') /Creator (anonymous) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))+ /Subject (unspecified) /Title (untitled) /Trapped /False+>>+endobj+7 0 obj+<<+/Count 1 /Kids [ 4 0 R ] /Type /Pages+>>+endobj+8 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 134+>>+stream+Gap(<YmuC?$jCj)`VT^mK4C@X![a4*)7gI0Cf+12IqSkRFOM8JrSbq#JS.<a0\@TbY]PK(@SZQl&DZ)2_;HHp*2m5u;;WsL2DRc[)*T5+Cs%o"^DD%hn2cG8C@i^s%reZhc2~>endstream+endobj+xref+0 9+0000000000 65535 f+0000000061 00000 n+0000000092 00000 n+0000000199 00000 n+0000004971 00000 n+0000005227 00000 n+0000005295 00000 n+0000005556 00000 n+0000005615 00000 n+trailer+<<+/ID+[<1c178198fbdfa51b25995d89d4102043><1c178198fbdfa51b25995d89d4102043>]+% ReportLab generated PDF document -- digest (opensource)++/Info 6 0 R+/Root 5 0 R+/Size 9+>>+startxref+5839+%%EOFdiff --git a/apps/api/src/__tests__/snips/fixtures/parse-tables.pdf b/apps/api/src/__tests__/snips/fixtures/parse-tables.pdfnew file mode 100644index 000000000..994fca107--- /dev/null+++ b/apps/api/src/__tests__/snips/fixtures/parse-tables.pdf@@ -0,0 +1,74 @@+%PDF-1.4+%���� ReportLab Generated PDF document (opensource)+1 0 obj+<<+/F1 2 0 R /F2 3 0 R+>>+endobj+2 0 obj+<<+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font+>>+endobj+3 0 obj+<<+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font+>>+endobj+4 0 obj+<<+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+5 0 obj+<<+/PageMode /UseNone /Pages 7 0 R /Type /Catalog+>>+endobj+6 0 obj+<<+/Author (Firecrawl tests) /CreationDate (D:20000101000000+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))+ /Subject (\(unspecified\)) /Title (parse-tables.pdf) /Trapped /False+>>+endobj+7 0 obj+<<+/Count 1 /Kids [ 4 0 R ] /Type /Pages+>>+endobj+8 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 682+>>+stream+Gat$ud>j\M&:EXkp@2=cY1M;C*a>UFWN)skaq&HN05sN+"k"fKhmG0T+FnQk&tJLYGZ\QDi:K>0qr\7S220A;#UggD!L-8Eq@gT>_;>5U;o]FqK#iUbLXl9Pb4&,^C6#)6!+W&RK[mk70W?@8V`T.URXJ%uf`?P7#LgK>S>s1!!9l#0+5t\)98@HL1H%Lf.gACSJ8U"OgMd?<dd6d;BYVZi9*:/8q"Y9)ZN?Jfboq^f?(Fc'#J%s"VImN<=VFYC9ajlu&ZB6[S*S)0%-\3@bE-_?5]Q,EpL\C8J;Hk9P)uW!UUokF0NH3R(aM/:i'P`\KUhP]QoAmB`49J`.hJDTr^@>P?a:>"lB`lmi(IrWZr##rV!gY%K^-3tC;Qe=]HgfHZQgV'S_5CfK>+Pg@;`gbN,JA&Pk7HR3't^BJqE>CmNFb\#*a"H;Fg*S,s=kY^,Cp47Z/=_RHcJ0go@KP]tR&Flp,pjMOXJK/'Xt([]!@Mn$VDSj,9dOTZmqYeE<Q[UASZH[]6O3Z_XD0S#+cWCW=Km/4C99Un<]<CLJA)3aniPM48Qe[RGOk:\H5A)HB!^?V[N\=\PWNTk?tPc\dh3X2@I-a^k(U>!F6X;%:p.qWVr_/rF@40BeE4^FT.Ih*p3fMp^uOn*40I/N+'P`MUJ\fIcl)&9!/+`Zb*f@75N>~>endstream+endobj+xref+0 9+0000000000 65535 f+0000000061 00000 n+0000000102 00000 n+0000000209 00000 n+0000000321 00000 n+0000000514 00000 n+0000000582 00000 n+0000000867 00000 n+0000000926 00000 n+trailer+<<+/ID+[<bc7348e251db46c3817e81e554985586><bc7348e251db46c3817e81e554985586>]+% ReportLab generated PDF document -- digest (opensource)++/Info 6 0 R+/Root 5 0 R+/Size 9+>>+startxref+1698+%%EOFdiff --git a/apps/api/src/__tests__/snips/fixtures/parse-text.pdf b/apps/api/src/__tests__/snips/fixtures/parse-text.pdfnew file mode 100644index 000000000..a37549ea4--- /dev/null+++ b/apps/api/src/__tests__/snips/fixtures/parse-text.pdf@@ -0,0 +1,74 @@+%PDF-1.4+%���� ReportLab Generated PDF document (opensource)+1 0 obj+<<+/F1 2 0 R /F2 3 0 R+>>+endobj+2 0 obj+<<+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font+>>+endobj+3 0 obj+<<+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font+>>+endobj+4 0 obj+<<+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+5 0 obj+<<+/PageMode /UseNone /Pages 7 0 R /Type /Catalog+>>+endobj+6 0 obj+<<+/Author (Firecrawl tests) /CreationDate (D:20000101000000+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))+ /Subject (\(unspecified\)) /Title (parse-text.pdf) /Trapped /False+>>+endobj+7 0 obj+<<+/Count 1 /Kids [ 4 0 R ] /Type /Pages+>>+endobj+8 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 515+>>+stream+GasajbAQ&g&A7ljk3*M]C9!;(?OJ]g4qKtEcYIO&Y`VJ!8MSTLGK%Q&d\]K"5S*;Lm*:R+7ZT"EFoX]aG^jL:C,@<*D7)h()5!AJc]rcBhX%L/e-ad=9t?I*V/>@_8UCYQWF>K.93j1jk/4>_ghaCR=KLs<P/Pep\96S2Hjtqm'q-4XhVd2tX:E"oo5]E<`<FK7#-)03FHd6cbrm:HP'KN+)Bdf>Z"t$Ff?OqJr^;/FC*gfSUj*tL?fI3UTT++-kIXWLA,2'W1&)+58HkY2XF&7J7lXkK_N?,L-W"9lE<[QrbG]ufF8dc4gOc60AK<@RXEG&aOmX#Tp,E4)#kl@H@q9Sq<p,%;\obVrqU]Stq7lF-`O^K&mb;^n5fj0O09*V.>U9>>0OJ'4P_.+=3jB8Bij.iENrb<c%hVL`4'i=<Il;E`c7Vd&ZW8S-kltkj8^t!%#YcWtfu78<I#_g'@&I&iQ&:SFX6H%+3em??NX=YI)TSFZM.KZ_pnuMAr;p'PkkY~>endstream+endobj+xref+0 9+0000000000 65535 f+0000000061 00000 n+0000000102 00000 n+0000000209 00000 n+0000000321 00000 n+0000000514 00000 n+0000000582 00000 n+0000000865 00000 n+0000000924 00000 n+trailer+<<+/ID+[<d97b4bc750377155affba2e76d870e81><d97b4bc750377155affba2e76d870e81>]+% ReportLab generated PDF document -- digest (opensource)++/Info 6 0 R+/Root 5 0 R+/Size 9+>>+startxref+1529+%%EOFdiff --git a/apps/api/src/__tests__/snips/v2/parse-fire-pdf.test.ts b/apps/api/src/__tests__/snips/v2/parse-fire-pdf.test.tsnew file mode 100644index 000000000..16194c2d3--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/parse-fire-pdf.test.ts@@ -0,0 +1,169 @@+import { readFileSync } from "node:fs";+import path from "node:path";+import { describeIf } from "../lib";+import { config } from "../../../config";+import { idmux, Identity, parse, parseWithFailure, scrapeTimeout } from "./lib";++function fixture(name: string) {+ return {+ content: readFileSync(path.join(__dirname, "..", "fixtures", name)),+ filename: name,+ contentType: "application/pdf",+ };+}++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "parse-fire-pdf",+ concurrency: 100,+ credits: 1000000,+ });+}, 10000 + scrapeTimeout);++const FIRE_PDF_ON =+ !process.env.TEST_SUITE_SELF_HOSTED &&+ !!config.FIRE_PDF_BASE_URL &&+ !!config.FIRE_PDF_ENABLE &&+ config.PARSE_PDF_ALWAYS_FIRE_PDF;++describeIf(FIRE_PDF_ON)("/v2/parse sends PDFs to FirePDF", () => {+ it.concurrent(+ "serves a simple text PDF from FirePDF, not direct extraction",+ async () => {+ const file = fixture("parse-text.pdf");+ const [plain, forced] = await Promise.all([+ parse({ options: { formats: ["markdown"] }, file }, identity),+ parse(+ {+ options: {+ formats: ["markdown"],+ parsers: [{ type: "pdf", blocks: true }],+ },+ file,+ },+ identity,+ ),+ ]);++ expect(plain.markdown).toContain("Quarterly Field Notes");+ expect(plain.markdown).toContain("Migratory birds");+ // Blocks always run FirePDF and leave the document markdown as is, so+ // a default parse served by FirePDF returns the same markdown.+ expect(plain.markdown).toBe(forced.markdown);+ expect(plain.metadata.numPages).toBe(1);+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "OCRs a scanned PDF in auto mode",+ async () => {+ const result = await parse(+ {+ options: { formats: ["markdown"] },+ file: fixture("parse-scanned.pdf"),+ },+ identity,+ );++ expect(result.markdown).toContain("Valve 7");+ },+ scrapeTimeout * 2,+ );++ it.concurrent(+ "keeps direct extraction for fast mode",+ async () => {+ const result = await parse(+ {+ options: {+ formats: ["markdown"],+ parsers: [{ type: "pdf", mode: "fast" }],+ },+ file: fixture("parse-text.pdf"),+ },+ identity,+ );++ expect(result.markdown).toContain("Quarterly Field Notes");+ },+ scrapeTimeout,+ );++ it.concurrent(+ "fails cleanly for a corrupt PDF",+ async () => {+ const response = await parseWithFailure(+ {+ options: { formats: ["markdown"] },+ file: {+ content: Buffer.from("%PDF-1.7\n%not a real document body\n"),+ filename: "corrupt.pdf",+ contentType: "application/pdf",+ },+ },+ identity,+ );++ expect(response.success).toBe(false);+ expect(response.code).toBeDefined();+ },+ scrapeTimeout,+ );+});++describeIf(FIRE_PDF_ON && !!config.PDF_RUST_EXTRACT_ENABLE)(+ "/v2/parse fast mode on scanned PDFs",+ () => {+ it.concurrent(+ "still refuses OCR in fast mode",+ async () => {+ const response = await parseWithFailure(+ {+ options: {+ formats: ["markdown"],+ parsers: [{ type: "pdf", mode: "fast" }],+ },+ file: fixture("parse-scanned.pdf"),+ },+ identity,+ );++ expect(response.success).toBe(false);+ expect(response.code).toBe("SCRAPE_PDF_OCR_REQUIRED");+ },+ scrapeTimeout,+ );+ },+);++describeIf(FIRE_PDF_ON && config.PARSE_FIRE_PDF_TABLE_FORMAT === "dynamic")(+ "/v2/parse dynamic table format",+ () => {+ it.concurrent(+ "returns merged-cell tables as HTML and simple tables as pipe tables",+ async () => {+ const result = await parse(+ {+ options: { formats: ["markdown"] },+ file: fixture("parse-tables.pdf"),+ },+ identity,+ );+ const markdown = result.markdown ?? "";+ const tables = markdown.match(/<table[\s\S]*?<\/table>/gi) ?? [];++ expect(tables).toHaveLength(1);+ expect(tables[0]).toMatch(/colspan=["']?2/i);+ expect(tables[0]).toContain("Region");+ expect(tables[0]).toContain("2025");+ expect(tables[0]).not.toContain("Apples");+ expect(markdown).toMatch(/\|\s*Fruit\s*\|\s*Quantity\s*\|/);+ expect(markdown).toMatch(/\|\s*Apples\s*\|\s*42\s*\|/);+ },+ scrapeTimeout * 2,+ );+ },+);diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 44d37c1ca..9747e0367 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -436,6 +436,18 @@ const configSchema = z.object({// cache service when one is configured; beyond the budget the request is// served normally. 0 disables the option.FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),+ // /v2/parse sends every PDF (auto/ocr mode) to FirePDF: no direct+ // text-layer extraction, no FIRE_PDF_PERCENT sampling, no MinerU+ // diversion. Explicit `mode: "fast"` keeps direct extraction. FirePDF+ // failures still fall back to MinerU / pdf-parse. Needs FIRE_PDF_ENABLE.+ PARSE_PDF_ALWAYS_FIRE_PDF: z.stringbool().default(true),+ // Table rendering FirePDF applies to /v2/parse documents. `dynamic` keeps+ // pipe tables where they are lossless and HTML where structure needs it+ // (merged cells, large tables); `off` sends no format (pipe tables).+ // Any explicit format puts the request outside the FirePDF content cache.+ PARSE_FIRE_PDF_TABLE_FORMAT: z+ .enum(["off", "markdown", "html", "dynamic"])+ .default("dynamic"),// Raster image OCR of image URLs and parse uploads through FirePDF (see// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.IMAGE_OCR_ENABLED: z.stringbool().default(false),diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.tsindex 4b62c4a63..322cc52c1 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts@@ -3,6 +3,8 @@ import {reconcilePageCountWithFirePdf,scrapePDFWithFirePDF,} from "../firePDF";+import { buildFirePdfJobOptions } from "../fire-pdf/utils";+import { config } from "../../../../../config";vi.mock("../../../lib/fetch", () => ({robustFetch: vi.fn(),@@ -426,3 +428,60 @@ describe("scrapePDFWithFirePDF page markers", () => {);});});++describe("FirePDF table format", () => {+ const originalTableFormat = config.PARSE_FIRE_PDF_TABLE_FORMAT;++ beforeEach(() => {+ vi.clearAllMocks();+ mockedRobustFetch.mockResolvedValue({+ markdown: "Document",+ failed_pages: null,+ pages_processed: 1,+ } as any);+ });++ afterEach(() => {+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = originalTableFormat;+ });++ it.each([+ ["parse", { isParse: true }, "dynamic", "dynamic"],+ ["parse with html", { isParse: true }, "html", "html"],+ ["parse with the format off", { isParse: true }, "off", undefined],+ ["scrape", {}, "dynamic", undefined],+ ] as const)(+ "sends the configured format for %s",+ async (_name, internalOptions, configured, expected) => {+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = configured;+ const meta = makeMeta();+ meta.internalOptions = { ...meta.internalOptions, ...internalOptions };++ await scrapePDFWithFirePDF(meta, "BASE64", 1);++ const body = mockedRobustFetch.mock.calls[0][0].body as Record<+ string,+ unknown+ >;+ expect(body.tableFormat).toBe(expected);+ expect(Object.hasOwn(body, "tableFormat")).toBe(expected !== undefined);+ },+ );++ it("puts the format in async job options only when set", () => {+ const base = {+ maxPages: undefined,+ pagesProcessed: 2,+ mode: "auto" as const,+ includePageMarkdown: false,+ includeBlocks: false,+ pageMarkers: false,+ };+ expect(buildFirePdfJobOptions({ ...base, tableFormat: "dynamic" })).toEqual(+ { pages_estimate: 2, mode: "auto", tableFormat: "dynamic" },+ );+ expect(buildFirePdfJobOptions({ ...base, tableFormat: undefined })).toEqual(+ { pages_estimate: 2, mode: "auto" },+ );+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.tsindex 50ea24001..1da0679e5 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts@@ -1842,3 +1842,72 @@ describe("FirePDF cache through the lookup service", () => {});});});++describe("FirePDF cache and parse table formats", () => {+ const originalTableFormat = config.PARSE_FIRE_PDF_TABLE_FORMAT;++ function parseMeta() {+ const meta = makeMeta();+ meta.internalOptions.isParse = true;+ return meta;+ }++ async function lookupAndSave(meta: any) {+ const cached = await tryGetCached(+ meta,+ "BASE64",+ "auto",+ undefined,+ 1,+ false,+ false,+ );+ await maybeSaveResult({+ meta,+ base64Content: "BASE64",+ mode: "auto",+ maxPages: undefined,+ includePageMarkdown: false,+ includeBlocks: false,+ result: { markdown: "fresh", html: "<p>fresh</p>", pagesProcessed: 1 },+ });+ return cached;+ }++ beforeEach(() => {+ vi.clearAllMocks();+ getCached.mockResolvedValue({ markdown: "cached", html: "<p>cached</p>" });+ saveCached.mockResolvedValue("saved-key");+ });++ afterEach(() => {+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = originalTableFormat;+ });++ it("neither reads nor writes for a parse request with a table format", async () => {+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = "dynamic";++ expect(await lookupAndSave(parseMeta())).toBeNull();+ expect(getCached).not.toHaveBeenCalled();+ expect(saveCached).not.toHaveBeenCalled();+ });++ it("keeps the cache for parse when the table format is off", async () => {+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = "off";++ expect(await lookupAndSave(parseMeta())).toMatchObject({+ markdown: "cached",+ });+ expect(getCached).toHaveBeenCalled();+ expect(saveCached).toHaveBeenCalled();+ });++ it("keeps the cache for scrape requests", async () => {+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = "dynamic";++ expect(await lookupAndSave(makeMeta())).toMatchObject({+ markdown: "cached",+ });+ expect(saveCached).toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.tsnew file mode 100644index 000000000..daf150742--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts@@ -0,0 +1,209 @@+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import path from "node:path";+import { detectPdf, processPdf } from "@mendable/firecrawl-rs";+import { config } from "../../../../../config";+import { scrapePDFWithFirePDF } from "../firePDF";+import { scrapePDFWithParsePDF } from "../pdfParse";+import { scrapePDF } from "../index";++vi.mock("@mendable/firecrawl-rs", () => ({+ processPdf: vi.fn(),+ detectPdf: vi.fn(),+}));++vi.mock("../firePDF", async importOriginal => ({+ ...(await importOriginal<typeof import("../firePDF")>()),+ scrapePDFWithFirePDF: vi.fn(),+}));++vi.mock("../pdfParse", () => ({+ scrapePDFWithParsePDF: vi.fn(),+}));++vi.mock("../markdownToHtml", () => ({+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),+}));++const mockedProcessPdf = vi.mocked(processPdf);+const mockedDetectPdf = vi.mocked(detectPdf);+const mockedFirePdf = vi.mocked(scrapePDFWithFirePDF);+const mockedPdfParse = vi.mocked(scrapePDFWithParsePDF);++const configKeys = [+ "PDF_RUST_EXTRACT_ENABLE",+ "FIRE_PDF_ENABLE",+ "FIRE_PDF_BASE_URL",+ "FIRE_PDF_PERCENT",+ "FIRE_PDF_ASYNC_PERCENT",+ "MINERU_PERCENT",+ "RUNPOD_MU_API_KEY",+ "RUNPOD_MU_POD_ID",+ "PARSE_PDF_ALWAYS_FIRE_PDF",+] as const;+const originalConfig = Object.fromEntries(+ configKeys.map(key => [key, config[key]]),+);++function setConfig(+ overrides: Partial<Record<(typeof configKeys)[number], unknown>>,+) {+ Object.assign(config as any, overrides);+}++const tempDirs: string[] = [];++async function makeMeta(+ internalOptions: Record<string, unknown>,+ parsers: unknown[] = ["pdf"],+) {+ const dir = await mkdtemp(path.join(tmpdir(), "parse-fire-pdf-"));+ tempDirs.push(dir);+ const filePath = path.join(dir, "doc.pdf");+ await writeFile(filePath, "%PDF-1.7\n%test\n");+ const logger: any = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ };+ logger.child = vi.fn(() => logger);+ return {+ id: "01a00000-0000-7000-8000-000000000000",+ url: "https://parse.firecrawl.dev/uploads/doc.pdf",+ options: { parsers },+ logger,+ featureFlags: new Set(["pdf"]),+ pdfPrefetch: {+ filePath,+ url: "https://parse.firecrawl.dev/uploads/doc.pdf",+ status: 200,+ proxyUsed: "basic",+ },+ abort: {+ throwIfAborted: vi.fn(),+ asSignal: vi.fn(() => new AbortController().signal),+ scrapeTimeout: vi.fn(() => 60_000),+ },+ internalOptions: { teamId: "team-test", ...internalOptions },+ } as any;+}++const eligibleTextPdf = {+ pdfType: "TextBased",+ confidence: 0.99,+ isComplex: false,+ pageCount: 1,+ markdown: "direct extraction",+ title: null,+ pagesNeedingOcr: [],+ logs: [],+};++describe("PDF routing for /v2/parse", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ setConfig({+ PDF_RUST_EXTRACT_ENABLE: true,+ FIRE_PDF_ENABLE: true,+ FIRE_PDF_BASE_URL: "http://fire-pdf.test",+ FIRE_PDF_PERCENT: 0,+ FIRE_PDF_ASYNC_PERCENT: 0,+ MINERU_PERCENT: 0,+ RUNPOD_MU_API_KEY: undefined,+ RUNPOD_MU_POD_ID: undefined,+ PARSE_PDF_ALWAYS_FIRE_PDF: true,+ });+ mockedProcessPdf.mockResolvedValue(eligibleTextPdf as any);+ mockedDetectPdf.mockResolvedValue({+ pdfType: "TextBased",+ pageCount: 1,+ title: null,+ logs: [],+ } as any);+ mockedFirePdf.mockResolvedValue({+ markdown: "fire-pdf",+ html: "<p>fire-pdf</p>",+ pagesProcessed: 1,+ });+ mockedPdfParse.mockResolvedValue({+ markdown: "pdf-parse",+ html: "<p>pdf-parse</p>",+ });+ });++ afterEach(async () => {+ await Promise.all(+ tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })),+ );+ });++ afterAll(() => {+ Object.assign(config as any, originalConfig);+ });++ it("sends a direct-extraction-eligible parse PDF to FirePDF regardless of FIRE_PDF_PERCENT", async () => {+ const result = await scrapePDF(await makeMeta({ isParse: true }));++ expect(result.markdown).toBe("fire-pdf");+ expect(mockedProcessPdf).not.toHaveBeenCalled();+ expect(mockedDetectPdf).toHaveBeenCalledTimes(1);+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ });++ it("keeps parse PDFs away from the MinerU diversion", async () => {+ setConfig({ MINERU_PERCENT: 100 });++ const result = await scrapePDF(await makeMeta({ isParse: true }));++ expect(result.markdown).toBe("fire-pdf");+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ });++ it("falls back to pdf-parse when FirePDF fails", async () => {+ mockedFirePdf.mockRejectedValueOnce(new Error("fire-pdf 503"));++ const result = await scrapePDF(await makeMeta({ isParse: true }));++ expect(result.markdown).toBe("pdf-parse");+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);+ expect(mockedPdfParse).toHaveBeenCalledTimes(1);+ });++ it("keeps direct extraction for scrape requests", async () => {+ setConfig({ FIRE_PDF_PERCENT: 100 });++ const result = await scrapePDF(await makeMeta({}));++ expect(result.markdown).toBe("direct extraction");+ expect(mockedProcessPdf).toHaveBeenCalledTimes(1);+ expect(mockedFirePdf).not.toHaveBeenCalled();+ });++ it("keeps direct extraction for parse in fast mode", async () => {+ const result = await scrapePDF(+ await makeMeta({ isParse: true }, [{ type: "pdf", mode: "fast" }]),+ );++ expect(result.markdown).toBe("direct extraction");+ expect(mockedFirePdf).not.toHaveBeenCalled();+ });++ it("restores direct extraction for parse when the flag is off", async () => {+ setConfig({ PARSE_PDF_ALWAYS_FIRE_PDF: false });++ const result = await scrapePDF(await makeMeta({ isParse: true }));++ expect(result.markdown).toBe("direct extraction");+ expect(mockedFirePdf).not.toHaveBeenCalled();+ });++ it("keeps direct extraction for parse when FirePDF is not configured", async () => {+ setConfig({ FIRE_PDF_ENABLE: false, FIRE_PDF_BASE_URL: undefined });++ const result = await scrapePDF(await makeMeta({ isParse: true }));++ expect(result.markdown).toBe("direct extraction");+ expect(mockedFirePdf).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.tsindex 9bd140119..39b13a047 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts@@ -10,6 +10,7 @@ import {import { cacheKeyShape, tryGetCached } from "./cache";import { lookupAdoptableFirePdfJob } from "./lookup";import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";+import { firePdfTableFormat } from "./request-metadata";type FirePdfByReferenceAttemptArgs = {meta: Meta;@@ -145,6 +146,7 @@ export async function runFirePdfByReferenceAttempt(includePageMarkdown,includeBlocks,pageMarkers,+ tableFormat: firePdfTableFormat(meta),}),);meta.abort.throwIfAborted();diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.tsindex 4014eb5d8..22b42e37c 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts@@ -13,6 +13,7 @@ import {import { deterministicPercentage } from "./routing";import { storage } from "../../../../../lib/gcs-jobs";import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";+import { parseAlwaysFirePdf } from "./request-metadata";/** Input handle for a FirePDF async submit that travels by GCS reference* instead of inline base64. Produced by {@link uploadPdfInputForFirePdf}. */@@ -199,10 +200,12 @@ export function largePdfLimitBytes(meta: Meta): number {* id (same distribution as a random draw) precisely so every gate — the* fire-engine handoff grant, the download admission, and the PDF engine's* routing — sees the SAME verdict; a per-call random draw would let them- * drift. Keyed apart from the async-cohort hash. */+ * drift. Keyed apart from the async-cohort hash. Parse requests that must+ * reach FirePDF (parseAlwaysFirePdf) are never diverted. */export function mineruDiverted(meta: Meta): boolean {return (config.MINERU_PERCENT > 0 &&+ !parseAlwaysFirePdf(meta) &&deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT);}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.tsindex 170b2daed..af8976c9d 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts@@ -20,6 +20,7 @@ import {firePdfCacheRefusedWritesTotal,} from "./metrics";import { cacheServiceConfigured, lookupCachedResult } from "./cache-service";+import { firePdfTableFormat } from "./request-metadata";import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";import {firePdfBlockPagesSchema,@@ -191,6 +192,9 @@ export async function tryGetCached(pageMarkers = false,): Promise<PDFProcessorResult | null> {if (meta.internalOptions.zeroDataRetention) return null;+ // Cache entries hold FirePDF's default table rendering; FirePDF itself+ // neither reads nor writes the cache for an explicit tableFormat.+ if (firePdfTableFormat(meta) !== undefined) return null;const viaService = cacheServiceConfigured();// No service and no bucket, no cache: nothing to hash, no refresh budget to spend.if (!viaService && !pdfCacheConfigured()) return null;@@ -412,6 +416,7 @@ export async function maybeSaveResult(args: {failedPages,} = args;if (meta.internalOptions.zeroDataRetention) return;+ if (firePdfTableFormat(meta) !== undefined) return;// With the lookup service in use, fire-pdf writes the result itself.if (cacheServiceConfigured()) return;if (!pdfCacheConfigured()) return;diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.tsindex 07b909ee6..ea46723f9 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts@@ -1,4 +1,6 @@import type { Meta } from "../../..";+import { config } from "../../../../../config";+import { getPDFMode } from "../../../../../controllers/v2/types";import { hasCustomRequestContext } from "../../../lib/request-context";/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */@@ -12,6 +14,29 @@ export function firePdfRequestKind(meta: Meta): "scrape" | "parse" {: "scrape";}+/** Whether this PDF must go to FirePDF instead of direct extraction, the+ * FIRE_PDF_PERCENT sample, or the MinerU diversion (see+ * PARSE_PDF_ALWAYS_FIRE_PDF). Fast mode keeps its text-layer-only contract. */+export function parseAlwaysFirePdf(meta: Meta): boolean {+ return (+ firePdfRequestKind(meta) === "parse" &&+ config.PARSE_PDF_ALWAYS_FIRE_PDF &&+ getPDFMode(meta.options.parsers) !== "fast" &&+ !!config.FIRE_PDF_ENABLE &&+ !!config.FIRE_PDF_BASE_URL+ );+}++/** The `tableFormat` sent to FirePDF, or undefined for its default (pipe+ * tables). Only parse requests set one (PARSE_FIRE_PDF_TABLE_FORMAT). */+export function firePdfTableFormat(+ meta: Meta,+): "markdown" | "html" | "dynamic" | undefined {+ if (firePdfRequestKind(meta) !== "parse") return undefined;+ const format = config.PARSE_FIRE_PDF_TABLE_FORMAT;+ return format === "off" ? undefined : format;+}+export function buildFirePdfRequestMetadata(meta: Meta,sourceKind: FirePdfSourceKind = "pdf",diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.tsindex 38a924820..540472b50 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts@@ -2,7 +2,10 @@ import type { Meta } from "../../..";import type { PDFMode } from "../../../../../controllers/v2/types";import { fetch as undiciFetch } from "undici";import { AbortManagerThrownError } from "../../../lib/abortManager";-import { buildFirePdfRequestMetadata } from "./request-metadata";+import {+ buildFirePdfRequestMetadata,+ firePdfTableFormat,+} from "./request-metadata";import {firePdfAsyncSubmit503Total,firePdfAsyncSubmitRetriesTotal,@@ -157,6 +160,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {includePageMarkdown,includeBlocks,pageMarkers,+ tableFormat: firePdfTableFormat(meta),}),};diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.tsindex 6e7e06027..82b0c67f7 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts@@ -158,6 +158,7 @@ export function buildFirePdfJobOptions(args: {includePageMarkdown: boolean;includeBlocks: boolean;pageMarkers: boolean;+ tableFormat: string | undefined;}): Record<string, unknown> {return {// fire-pdf accepts only a positive integer here. When the router has no@@ -179,6 +180,7 @@ export function buildFirePdfJobOptions(args: {// `page_markers`. Sending snake_case here would be rejected as an// unknown option....(args.pageMarkers && { pageMarkers: true }),+ ...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),};}diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.tsindex dd2ff5aee..dd6e7015a 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts@@ -14,6 +14,7 @@ import {import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";import {buildFirePdfRequestMetadata,+ firePdfTableFormat,type FirePdfSourceKind,} from "./fire-pdf/request-metadata";@@ -106,6 +107,7 @@ export async function scrapePDFWithFirePDF(const zdr = meta.internalOptions.zeroDataRetention === true;const pdfSha256 = createPdfCacheKey(base64Content);+ const tableFormat = firePdfTableFormat(meta);// Explicit deadline contract with fire-pdf (mirrors mineru-api):// timeout — remaining scrape-tier budget in ms (from AbortManager)@@ -141,6 +143,7 @@ export async function scrapePDFWithFirePDF(...(includePageMarkdown && { include_page_markdown: true }),...(includeBlocks && { include_blocks: true }),...(pageMarkers && { page_markers: true }),+ ...(tableFormat && { tableFormat }),// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats// these as optional — older fire-pdf builds will ignore unknown fields.team_id: meta.internalOptions.teamId,diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.tsindex b43c37a94..138218220 100644--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts@@ -60,6 +60,7 @@ import {firePdfFeaturesLabel,recordFirePdfRoute,} from "./fire-pdf/routing";+import { parseAlwaysFirePdf } from "./fire-pdf/request-metadata";import { scrapePDFWithParsePDF } from "./pdfParse";import { toPublicBlocks } from "./blocks";import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";@@ -237,6 +238,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {pageMarkers) &&!!config.FIRE_PDF_BASE_URL;+ // /v2/parse: every non-fast PDF goes to FirePDF (PARSE_PDF_ALWAYS_FIRE_PDF).+ // Unlike forceFirePDF this keeps the MinerU / pdf-parse fallbacks.+ const alwaysFirePdf = parseAlwaysFirePdf(meta);+// The MinerU diversion is deterministic on the scrape id (see// mineruDiverted) so this routing verdict is BY CONSTRUCTION the same// one inside byReferenceReachableForRequest() — and the same one@@ -329,7 +334,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {});}- if (!rustEnabled || mode === "ocr" || forceFirePDF || routeToMinerU) {+ if (+ !rustEnabled ||+ mode === "ocr" ||+ forceFirePDF ||+ routeToMinerU ||+ alwaysFirePdf+ ) {// Legacy / OCR path: detect metadata only, skip Rust extraction.// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,// matching current prod behaviour (detectPdf → MinerU → pdfParse).@@ -360,6 +371,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {url: meta.rewrittenUrl ?? meta.url,rustEnabled,mode,+ alwaysFirePdf,});totalPageCount = detection.pageCount;@@ -657,7 +669,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {config.FIRE_PDF_ENABLE &&config.FIRE_PDF_BASE_URL &&fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&- Math.random() * 100 < config.FIRE_PDF_PERCENT));+ (alwaysFirePdf || Math.random() * 100 < config.FIRE_PDF_PERCENT)));if (useFirePDF) {// Async is a server-controlled cohort within traffic already selected