Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): send /v2/parse PDFs to FirePDF with dynamic table format (#5110)

Abimael Martell8604989d7526
diff --git a/apps/api/src/__tests__/snips/fixtures/parse-scanned.pdf b/apps/api/src/__tests__/snips/fixtures/parse-scanned.pdf
new file mode 100644
index 000000000..f146cde68
--- /dev/null
+++ b/apps/api/src/__tests__/snips/fixtures/parse-scanned.pdf
@@ -0,0 +1,79 @@
+%PDF-1.3
+%���� ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BitsPerComponent 8 /ColorSpace /DeviceRGB /Filter [ /ASCII85Decode /FlateDecode ] /Height 300 /Length 4581 /Subtype /Image
+ /Type /XObject /Width 850
+>>
+stream
+Gb"0SdB`&IXp%Z$s/3C<?+9*888MF,$t5.cfLqNCA7DLocR/\mzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!5NG6Nr8Y/595`ZQ_UYVkmGG\*#SbF%Y*i9r;u%dg4/ek(u0fi"5lWG-_IT'4)4fGf+j6tV,M>D!;s%AO#fN'DU#B!?fG-/!,5(j$%.B\;:emaTDdXin\R=B!1qjs"lCHsiiE*G3(a0kr#YV@SFY!FPQ7LbqBb\lZ>FlkkJpf74Qa/kB7DC9nT9c>DJ?EJoIiAMY33C!@H3C*p0h&tF]&-)G2V+\oVN_eK^#NgG;$WD<uO!*>'eVCPJd<4dm!/ORZ;&oQq3fVT>[Knl3F@J4PEG>Z!0V-a*P2?:@&qj"1N?j4$L>O#29=5:=HpV<Z3m)C4F[io$qQm;2kNYI'd::D./<IEinW=7oH`Wf1T8O>K#fe9@(5WH`km(o@Lk^2q@nZ:G7^c?ZXHs`R?MnL5i<e`PGT-ZM2S*@):KX4M<Nm(BhWLVfW]O,u:eriT!V(Z2SG$&PX?a3iK%.*FJfhZnf.0ACDLZT1/nMX7le$GDiQFh5HTVNNF,B'$6&%]Q6pB^kjb1;$Y;0a0E"[pFNbDIaIB5[8/$7m<8;XgAuN*FK<@W[-'R,W$TZtZE_qQc+)<6>Ap=1D1ej$I_87-Gr?\rH`H#1Ur,%OM2s'JXZhD[pi5h%bu2D3ae.=0:^5_#qiqf/):&(`)Yb2^GiO)7GNfjE`l3d*oqU1uW0Y60;0V]4\&rQ-<a9GUWU)\E\[email protected]*Uql@[@Y,VDVQK%7Gr`'gkcB64:Y4!AAENQlOX>_?Xs7/1[$l%nbk/*>>;r.&:pA:+?^74B+,971hfVW.f6RQn(N.N$cP*r=2n+*9;X)YQ3Zf5=/p#X#\rJ>nC8q6CGat]+aGR*OE][;O)a=b3h*.":jd:R&q!U(iS2aa$HU"%f*XT0;Vt&0oXV0X*XJQ@$aNW,TQduY(f+7/Ve[AB^(9ckgf!e>"aV?mt/)s/"mc4+Ea1b^fr/BkLep'F/n'[Od9:K8ACA$!"*]*B^Q28Fo"UfA,`m\T-F&p&):o>Jk:Ee/!:moB\Y.sUNMG;eP[d)W<:]%6]$iHC^4dNTRK"D`#e"qDsIT=Bl<M":];PLo)FBt$g1T?$Fe5$1bj/ke^oXT9PX`iG`MO&)-iu\)tr6midHJnOfh#f8'_t9O*g90P/ltt9qMgjsbhDt26il$U_g4,3^CNPQ<EmN:!B3]r6l)4.6G3kc%HEZtl4&6jkH"Zk@bC,bff`m_$2elEp`c$j55$e'RS9\XRnC1L:e<2W%`MH:\,?eh^n$_Tcfpf,<_iRM+Y<4B(4`TEC7BXRTo1FsL6%&"<<n"efV'B9WDss?b_@/Vfob"Tj/eP(Zc^k_K\j_JiH.'[lZE=_U;P^/K/'b*TH%A%lr">t[NtaNr]:r<q,#*Fm1iAa73)!>uF4d;S'@etRY#\19)F'51im"BngTd-Jn\^G]'sF_%msS<rNX#5CrJCk5J,HA"mCDuWXQq'ipM%:OGKtRbcXE$?&R@-,QD^l%pO7uXT71[u@V+STTeTQ[ZM2:ggKAit+9,O@lVd=HN7Dgpil5$8n.E@oC>#t<:rPPX2sE#U".\f0hP'8VVNkM2qLcrMP;[k,5+]M>*cN@7Mu'N(ol_6)&&sE)NakJ7)d>JW^GmFd%dm?0^phS$pXJ26duB5T[bC"_3(lj.Nu#spFo$*;F,3n(nR8AoMH;^-3,gs9)k_GP7/h:1E#9g_jK.:Do@9tgSn2(=kqh0?p^!#7l%6ftX+,OUI;YC+'.FE7Mt`Vl*s@V2Rc]Hf.$lD]*9hN5Ac)\OLDW%ocfQ>IWH^\D[s<JDV]'pEeijn"'eBO&[+4H*:0Vk#.qm\1lGFK5@&Zq?W^No+o(\_Z_S#U\QN^=\2JIYups@;iI5$.*3H/d,lRU@=^%T-Og1Ml%eZI,SF0h>/=j8&D-s^eZH2Q:dE8G@[6XC,NdOMqR4G8%,BnrOu,+TZsCs4f/=JX%tkFZVpr.+J[+PgeDE8#8Fa)Q*/X7;AFbf@n2Nq0`)[email protected]#-B?mcCT5mR&0Xes%8KOCl)Qa&&U?e:S0THHjS\>!!%M%?[VnSIt:sT!!#70^[M:)r"4DN!<E$[s/uq5i/dB.%fkN@n>d'"zzzzzzzzzzzzzzzzzzzzzzzzz!!!"L]6jIj:BuFTO8nb7/-,T.O+5]L>SjTQo*7Fa4rp.CV[:.O'u,SXr?D-\Y=QG)r[athIQ.5Hh)%$;5JR%Qo;h?+Egl?=duW9YA&Blt^5Vd`lMI(>rZ?Xfhr?QHn#"^X9k2'mUu/AV?d(b6f":[*N!!Y+RssF9)B3SQnZU`b97pNtr6(oIKs0T'g$GRX\XUTn)bI\ROX6&5Q_nLpN8:(6_\TeB\d$^[3Vi$JN&t9jVC`$meh+W#rlMW[BG]!9FoGjrHG!L<ml=#88VoBPYiP:2:YZnOrq3Wkab/?uoksWSWq_P%.HSi$dAeKabHjSt/2`CQ(#QUZD<Z=ucT#bsmZUtkPV_05S19YNROo@<[nAkj@eY1b+DCG%3id1HD$Q>3BeN(be@X=*\)IKJ:8umG--<SBmg"t2kZLSCkk;!L%@VAj+!`rg;9JsOAf-Jtk[0C;lVQIT#[2PJ#0#UL95e]YQf.0o/Y31$H/Z.!'tH5T8(2Up`IYLg=%e`sg&'F"h52A*j4aaL`B-T`/N.<JI[2Zo]K3iSreSHkOf0l%dN'gBJL*rBe=a=]r3.aBf;7k8o3?V)>]I:XHUc/hcF<S!p%`\MH=F+t@r4?QGhIF`V<kOd^N:%#",u^*rb5N;YJdIhDg?M6/:PnZDktpAjjLM;gt3A/><LbI^&,1YT$L3:8S,N:%q/`3BeFOTDBISIk<tNnG3K%Y[glU/:lk@5fK_dmD;Fn`a7%d*?Mj3MrEJ?SZ":t%T(_9H*<sK>%q4L7>>@!K#aLK9F8As[9,I'SnZ6tbC`k>#Y&/NblNNE1<hJL%]BH!$m^elPh5%R"k2g7K97=A]\+_`>h7A?,BQ__nV#g#K<idrtk_ci02NZMb*DtVUflZ,3CD)\Ll#kK5hFc+[%.USMG%k3q\tmbe_dHR3j5%;$pAFRIO0ahT>Q&?6GVs-I6W^qs!S,L4#b&JFnb;RgJk*V!@?q65`Y2Hs?1Xo$n`%]UMSlMH&&t2)cKai^LTU3:(?\ktLN_5uo&*S#Y88j'BUn&HG!"a7_cNEJ=>G>*,;4*!\iB%t?/!i-Ok=C"2.(4T41+6)\_3jHYNA(8>R;CLTCuuuh`S8:_ki,\/FP19[9>G)T$Dg_]]-9)4?F*/GhjQ^@`sc$dOLSAdNO`2P!%p%cDtf7,AiiHEPf9JM+9Pl):+_ODeUu0O*Bn.g/?7+Ur_9[(=UENg.:/E1iiCFNE:D"i_or!K=>mt_S3e7Oig(TQF+Js4YBI7e2]M:`kfBu]l\POW)rl'n(Y-^YZk3(ET[e/^\TKe"'jZkbK&XKI6^'RiCau4'B=#s;R]o3@$iUX?7rqT<h#]rpR^hq@A6,.H.T!9;_@FlFdeH-VDlcQI/8ufe)JAJ=88e?nakR$./:sq?cgB[d!NY5]:H?e/H<dS]K+)Kk<XObq8PKARLGL+NOj,6TA"Cd>KbBFXulTKJp=?\7SCQA`@\gqLuS`IF&qds]t\[<f._E^%#X'ShocXJ5BskHSY>.Yg<]NY-A1Dc"4)D>[VZp3hL)2??ZQ"&"'jZnDKWR^n]#qCS.b==HR0oG62\`Ir+fKERPl0u]$WsLcHnG.hXbTC]t1jli#hc^okk[DFnMZ>GIq^ajJ@s-BDDDaGV'.'FEr-XK==/,>oN0T)"28/](,2i"F.Qc(<?G3hQ\APqsqu<YF%oC1fs:`Dk29+Y\uN"kp*BdgcWL;(E[Yc@"Go0M;XPAgIeqIg9\2LqFKA6%JO(G/G4VQ?E<8;]DO?qUt`9&1eGkVGFdkh=^p\fIIKJ@K=.?NZpV&KOt,>^J*V`XVq^Tq\*\]r?B6=(9-OhWY1(q`e)dW+QaYk%Rnb:i[rG.?#T>Wo/73_gi`P1rM46D.g28cGI;"mkesAnL%d"__mVJZ_eqk:,p=^055-\Ki0K)`D]_BE=%\#k40"ppj)oY7/k;ii,mHQ^gUA/iHjl:6@-J'hfD\ZLr:DF9Xh`BCAOWh",B9t:ER>r]*f6'kdhlAnHf;#/Y!bo1]/b6>>g9,=!^DY&5YIJQFggfeW<1Y]tS8lI)SBo%dQfkq[qRdM1;FNL$DnXY;.77QX_6k(2O1,JA-b`mS5uuaCDU]^\\8_:XQ*ikPj[GZQVpi)Tq_hagHg[;=pXnF?`SKkCHe>-7il+L?>E,!$'>F)VCgHCn=MFh1MTZ6QcMfk2^:NbU5c*5`/s2.AnQHF$V$XrpiMuf%:(mUEn2oTq3)'D2N8FrV`+]8QL]!Y"06*E\^jt;K^Vjgf!,)4)pa&ZK!''&s"Fl9W"98Z(:A?,1'*&"4zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz!!!#oT`5+S_eP&~>endstream
+endobj
+4 0 obj
+<<
+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /XObject <<
+/FormXob.c71196c43e59bbc614ba2df60682674c 3 0 R
+>>
+>> /Rotate 0 /Trans <<
+
+>>
+ /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/PageMode /UseNone /Pages 7 0 R /Type /Catalog
+>>
+endobj
+6 0 obj
+<<
+/Author (anonymous) /CreationDate (D:20000101000000+00'00') /Creator (anonymous) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))
+ /Subject (unspecified) /Title (untitled) /Trapped /False
+>>
+endobj
+7 0 obj
+<<
+/Count 1 /Kids [ 4 0 R ] /Type /Pages
+>>
+endobj
+8 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 134
+>>
+stream
+Gap(<YmuC?$jCj)`VT^mK4C@X![a4*)7gI0Cf+12IqSkRFOM8JrSbq#JS.<a0\@TbY]PK(@SZQl&DZ)2_;HHp*2m5u;;WsL2DRc[)*T5+Cs%o"^DD%hn2cG8C@i^s%reZhc2~>endstream
+endobj
+xref
+0 9
+0000000000 65535 f
+0000000061 00000 n
+0000000092 00000 n
+0000000199 00000 n
+0000004971 00000 n
+0000005227 00000 n
+0000005295 00000 n
+0000005556 00000 n
+0000005615 00000 n
+trailer
+<<
+/ID
+[<1c178198fbdfa51b25995d89d4102043><1c178198fbdfa51b25995d89d4102043>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 6 0 R
+/Root 5 0 R
+/Size 9
+>>
+startxref
+5839
+%%EOF
diff --git a/apps/api/src/__tests__/snips/fixtures/parse-tables.pdf b/apps/api/src/__tests__/snips/fixtures/parse-tables.pdf
new file mode 100644
index 000000000..994fca107
--- /dev/null
+++ b/apps/api/src/__tests__/snips/fixtures/parse-tables.pdf
@@ -0,0 +1,74 @@
+%PDF-1.4
+%���� ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>>
+ /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/PageMode /UseNone /Pages 7 0 R /Type /Catalog
+>>
+endobj
+6 0 obj
+<<
+/Author (Firecrawl tests) /CreationDate (D:20000101000000+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))
+ /Subject (\(unspecified\)) /Title (parse-tables.pdf) /Trapped /False
+>>
+endobj
+7 0 obj
+<<
+/Count 1 /Kids [ 4 0 R ] /Type /Pages
+>>
+endobj
+8 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 682
+>>
+stream
+Gat$ud>j\M&:EXkp@2=cY1M;C*a>UFWN)skaq&HN05sN+"k"fKhmG0T+FnQk&tJLYGZ\QDi:K>0qr\7S220A;#UggD!L-8Eq@gT>_;>5U;o]FqK#iUbLXl9Pb4&,^C6#)6!+W&RK[mk70W?@8V`T.URXJ%uf`?P7#LgK>S>s1!!9l#0+5t\)98@HL1H%Lf.gACSJ8U"OgMd?<dd6d;BYVZi9*:/8q"Y9)ZN?Jfboq^f?(Fc'#J%s"VImN<=VFYC9ajlu&ZB6[S*S)0%-\3@bE-_?5]Q,EpL\C8J;Hk9P)uW!UUokF0NH3R(aM/:i'P`\KUhP]QoAmB`49J`.hJDTr^@>P?a:>"lB`lmi(IrWZr##rV!gY%K^-3tC;Qe=]HgfHZQgV'S_5CfK>+Pg@;`gbN,JA&Pk7HR3't^BJqE>CmNFb\#*a"H;Fg*S,s=kY^,Cp47Z/=_RHcJ0go@KP]tR&Flp,pjMOXJK/'Xt([]!@Mn$VDSj,9dOTZmqYeE<Q[UASZH[]6O3Z_XD0S#+cWCW=Km/4C99Un<]<CLJA)3aniPM48Qe[RGOk:\H5A)HB!^?V[N\=\PWNTk?tPc\dh3X2@I-a^k(U>!F6X;%:p.qWVr_/rF@40BeE4^FT.Ih*p3fMp^uOn*40I/N+'P`MUJ\fIcl)&9!/+`Zb*f@75N>~>endstream
+endobj
+xref
+0 9
+0000000000 65535 f
+0000000061 00000 n
+0000000102 00000 n
+0000000209 00000 n
+0000000321 00000 n
+0000000514 00000 n
+0000000582 00000 n
+0000000867 00000 n
+0000000926 00000 n
+trailer
+<<
+/ID
+[<bc7348e251db46c3817e81e554985586><bc7348e251db46c3817e81e554985586>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 6 0 R
+/Root 5 0 R
+/Size 9
+>>
+startxref
+1698
+%%EOF
diff --git a/apps/api/src/__tests__/snips/fixtures/parse-text.pdf b/apps/api/src/__tests__/snips/fixtures/parse-text.pdf
new file mode 100644
index 000000000..a37549ea4
--- /dev/null
+++ b/apps/api/src/__tests__/snips/fixtures/parse-text.pdf
@@ -0,0 +1,74 @@
+%PDF-1.4
+%���� ReportLab Generated PDF document (opensource)
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>>
+ /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/PageMode /UseNone /Pages 7 0 R /Type /Catalog
+>>
+endobj
+6 0 obj
+<<
+/Author (Firecrawl tests) /CreationDate (D:20000101000000+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20000101000000+00'00') /Producer (ReportLab PDF Library - \(opensource\))
+ /Subject (\(unspecified\)) /Title (parse-text.pdf) /Trapped /False
+>>
+endobj
+7 0 obj
+<<
+/Count 1 /Kids [ 4 0 R ] /Type /Pages
+>>
+endobj
+8 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 515
+>>
+stream
+GasajbAQ&g&A7ljk3*M]C9!;(?OJ]g4qKtEcYIO&Y`VJ!8MSTLGK%Q&d\]K"5S*;Lm*:R+7ZT"EFoX]aG^jL:C,@<*D7)h()5!AJc]rcBhX%L/e-ad=9t?I*V/>@_8UCYQWF>K.93j1jk/4>_ghaCR=KLs<P/Pep\96S2Hjtqm'q-4XhVd2tX:E"oo5]E<`<FK7#-)03FHd6cbrm:HP'KN+)Bdf>Z"t$Ff?OqJr^;/FC*gfSUj*tL?fI3UTT++-kIXWLA,2'W1&)+58HkY2XF&7J7lXkK_N?,L-W"9lE<[QrbG]ufF8dc4gOc60AK<@RXEG&aOmX#Tp,E4)#kl@H@q9Sq<p,%;\obVrqU]Stq7lF-`O^K&mb;^n5fj0O09*V.>U9>>0OJ'4P_.+=3jB8Bij.iENrb<c%hVL`4'i=<Il;E`c7Vd&ZW8S-kltkj8^t!%#YcWtfu78<I#_g'@&I&iQ&:SFX6H%+3em??NX=YI)TSFZM.KZ_pnuMAr;p'PkkY~>endstream
+endobj
+xref
+0 9
+0000000000 65535 f
+0000000061 00000 n
+0000000102 00000 n
+0000000209 00000 n
+0000000321 00000 n
+0000000514 00000 n
+0000000582 00000 n
+0000000865 00000 n
+0000000924 00000 n
+trailer
+<<
+/ID
+[<d97b4bc750377155affba2e76d870e81><d97b4bc750377155affba2e76d870e81>]
+% ReportLab generated PDF document -- digest (opensource)
+
+/Info 6 0 R
+/Root 5 0 R
+/Size 9
+>>
+startxref
+1529
+%%EOF
diff --git a/apps/api/src/__tests__/snips/v2/parse-fire-pdf.test.ts b/apps/api/src/__tests__/snips/v2/parse-fire-pdf.test.ts
new file mode 100644
index 000000000..16194c2d3
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/parse-fire-pdf.test.ts
@@ -0,0 +1,169 @@
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import { describeIf } from "../lib";
+import { config } from "../../../config";
+import { idmux, Identity, parse, parseWithFailure, scrapeTimeout } from "./lib";
+
+function fixture(name: string) {
+ return {
+ content: readFileSync(path.join(__dirname, "..", "fixtures", name)),
+ filename: name,
+ contentType: "application/pdf",
+ };
+}
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "parse-fire-pdf",
+ concurrency: 100,
+ credits: 1000000,
+ });
+}, 10000 + scrapeTimeout);
+
+const FIRE_PDF_ON =
+ !process.env.TEST_SUITE_SELF_HOSTED &&
+ !!config.FIRE_PDF_BASE_URL &&
+ !!config.FIRE_PDF_ENABLE &&
+ config.PARSE_PDF_ALWAYS_FIRE_PDF;
+
+describeIf(FIRE_PDF_ON)("/v2/parse sends PDFs to FirePDF", () => {
+ it.concurrent(
+ "serves a simple text PDF from FirePDF, not direct extraction",
+ async () => {
+ const file = fixture("parse-text.pdf");
+ const [plain, forced] = await Promise.all([
+ parse({ options: { formats: ["markdown"] }, file }, identity),
+ parse(
+ {
+ options: {
+ formats: ["markdown"],
+ parsers: [{ type: "pdf", blocks: true }],
+ },
+ file,
+ },
+ identity,
+ ),
+ ]);
+
+ expect(plain.markdown).toContain("Quarterly Field Notes");
+ expect(plain.markdown).toContain("Migratory birds");
+ // Blocks always run FirePDF and leave the document markdown as is, so
+ // a default parse served by FirePDF returns the same markdown.
+ expect(plain.markdown).toBe(forced.markdown);
+ expect(plain.metadata.numPages).toBe(1);
+ },
+ scrapeTimeout * 2,
+ );
+
+ it.concurrent(
+ "OCRs a scanned PDF in auto mode",
+ async () => {
+ const result = await parse(
+ {
+ options: { formats: ["markdown"] },
+ file: fixture("parse-scanned.pdf"),
+ },
+ identity,
+ );
+
+ expect(result.markdown).toContain("Valve 7");
+ },
+ scrapeTimeout * 2,
+ );
+
+ it.concurrent(
+ "keeps direct extraction for fast mode",
+ async () => {
+ const result = await parse(
+ {
+ options: {
+ formats: ["markdown"],
+ parsers: [{ type: "pdf", mode: "fast" }],
+ },
+ file: fixture("parse-text.pdf"),
+ },
+ identity,
+ );
+
+ expect(result.markdown).toContain("Quarterly Field Notes");
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "fails cleanly for a corrupt PDF",
+ async () => {
+ const response = await parseWithFailure(
+ {
+ options: { formats: ["markdown"] },
+ file: {
+ content: Buffer.from("%PDF-1.7\n%not a real document body\n"),
+ filename: "corrupt.pdf",
+ contentType: "application/pdf",
+ },
+ },
+ identity,
+ );
+
+ expect(response.success).toBe(false);
+ expect(response.code).toBeDefined();
+ },
+ scrapeTimeout,
+ );
+});
+
+describeIf(FIRE_PDF_ON && !!config.PDF_RUST_EXTRACT_ENABLE)(
+ "/v2/parse fast mode on scanned PDFs",
+ () => {
+ it.concurrent(
+ "still refuses OCR in fast mode",
+ async () => {
+ const response = await parseWithFailure(
+ {
+ options: {
+ formats: ["markdown"],
+ parsers: [{ type: "pdf", mode: "fast" }],
+ },
+ file: fixture("parse-scanned.pdf"),
+ },
+ identity,
+ );
+
+ expect(response.success).toBe(false);
+ expect(response.code).toBe("SCRAPE_PDF_OCR_REQUIRED");
+ },
+ scrapeTimeout,
+ );
+ },
+);
+
+describeIf(FIRE_PDF_ON && config.PARSE_FIRE_PDF_TABLE_FORMAT === "dynamic")(
+ "/v2/parse dynamic table format",
+ () => {
+ it.concurrent(
+ "returns merged-cell tables as HTML and simple tables as pipe tables",
+ async () => {
+ const result = await parse(
+ {
+ options: { formats: ["markdown"] },
+ file: fixture("parse-tables.pdf"),
+ },
+ identity,
+ );
+ const markdown = result.markdown ?? "";
+ const tables = markdown.match(/<table[\s\S]*?<\/table>/gi) ?? [];
+
+ expect(tables).toHaveLength(1);
+ expect(tables[0]).toMatch(/colspan=["']?2/i);
+ expect(tables[0]).toContain("Region");
+ expect(tables[0]).toContain("2025");
+ expect(tables[0]).not.toContain("Apples");
+ expect(markdown).toMatch(/\|\s*Fruit\s*\|\s*Quantity\s*\|/);
+ expect(markdown).toMatch(/\|\s*Apples\s*\|\s*42\s*\|/);
+ },
+ scrapeTimeout * 2,
+ );
+ },
+);
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 44d37c1ca..9747e0367 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -436,6 +436,18 @@ const configSchema = z.object({
// cache service when one is configured; beyond the budget the request is
// served normally. 0 disables the option.
FIRE_PDF_CACHE_REFRESH_PER_MINUTE: z.coerce.number().int().min(0).default(10),
+ // /v2/parse sends every PDF (auto/ocr mode) to FirePDF: no direct
+ // text-layer extraction, no FIRE_PDF_PERCENT sampling, no MinerU
+ // diversion. Explicit `mode: "fast"` keeps direct extraction. FirePDF
+ // failures still fall back to MinerU / pdf-parse. Needs FIRE_PDF_ENABLE.
+ PARSE_PDF_ALWAYS_FIRE_PDF: z.stringbool().default(true),
+ // Table rendering FirePDF applies to /v2/parse documents. `dynamic` keeps
+ // pipe tables where they are lossless and HTML where structure needs it
+ // (merged cells, large tables); `off` sends no format (pipe tables).
+ // Any explicit format puts the request outside the FirePDF content cache.
+ PARSE_FIRE_PDF_TABLE_FORMAT: z
+ .enum(["off", "markdown", "html", "dynamic"])
+ .default("dynamic"),
// Raster image OCR of image URLs and parse uploads through FirePDF (see
// lib/image-ocr-gate.ts). Needs FIRE_PDF_BASE_URL.
IMAGE_OCR_ENABLED: z.stringbool().default(false),
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
index 4b62c4a63..322cc52c1 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDF.test.ts
@@ -3,6 +3,8 @@ import {
reconcilePageCountWithFirePdf,
scrapePDFWithFirePDF,
} from "../firePDF";
+import { buildFirePdfJobOptions } from "../fire-pdf/utils";
+import { config } from "../../../../../config";
vi.mock("../../../lib/fetch", () => ({
robustFetch: vi.fn(),
@@ -426,3 +428,60 @@ describe("scrapePDFWithFirePDF page markers", () => {
);
});
});
+
+describe("FirePDF table format", () => {
+ const originalTableFormat = config.PARSE_FIRE_PDF_TABLE_FORMAT;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mockedRobustFetch.mockResolvedValue({
+ markdown: "Document",
+ failed_pages: null,
+ pages_processed: 1,
+ } as any);
+ });
+
+ afterEach(() => {
+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = originalTableFormat;
+ });
+
+ it.each([
+ ["parse", { isParse: true }, "dynamic", "dynamic"],
+ ["parse with html", { isParse: true }, "html", "html"],
+ ["parse with the format off", { isParse: true }, "off", undefined],
+ ["scrape", {}, "dynamic", undefined],
+ ] as const)(
+ "sends the configured format for %s",
+ async (_name, internalOptions, configured, expected) => {
+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = configured;
+ const meta = makeMeta();
+ meta.internalOptions = { ...meta.internalOptions, ...internalOptions };
+
+ await scrapePDFWithFirePDF(meta, "BASE64", 1);
+
+ const body = mockedRobustFetch.mock.calls[0][0].body as Record<
+ string,
+ unknown
+ >;
+ expect(body.tableFormat).toBe(expected);
+ expect(Object.hasOwn(body, "tableFormat")).toBe(expected !== undefined);
+ },
+ );
+
+ it("puts the format in async job options only when set", () => {
+ const base = {
+ maxPages: undefined,
+ pagesProcessed: 2,
+ mode: "auto" as const,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ pageMarkers: false,
+ };
+ expect(buildFirePdfJobOptions({ ...base, tableFormat: "dynamic" })).toEqual(
+ { pages_estimate: 2, mode: "auto", tableFormat: "dynamic" },
+ );
+ expect(buildFirePdfJobOptions({ ...base, tableFormat: undefined })).toEqual(
+ { pages_estimate: 2, mode: "auto" },
+ );
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
index 50ea24001..1da0679e5 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/firePDFCache.test.ts
@@ -1842,3 +1842,72 @@ describe("FirePDF cache through the lookup service", () => {
});
});
});
+
+describe("FirePDF cache and parse table formats", () => {
+ const originalTableFormat = config.PARSE_FIRE_PDF_TABLE_FORMAT;
+
+ function parseMeta() {
+ const meta = makeMeta();
+ meta.internalOptions.isParse = true;
+ return meta;
+ }
+
+ async function lookupAndSave(meta: any) {
+ const cached = await tryGetCached(
+ meta,
+ "BASE64",
+ "auto",
+ undefined,
+ 1,
+ false,
+ false,
+ );
+ await maybeSaveResult({
+ meta,
+ base64Content: "BASE64",
+ mode: "auto",
+ maxPages: undefined,
+ includePageMarkdown: false,
+ includeBlocks: false,
+ result: { markdown: "fresh", html: "<p>fresh</p>", pagesProcessed: 1 },
+ });
+ return cached;
+ }
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getCached.mockResolvedValue({ markdown: "cached", html: "<p>cached</p>" });
+ saveCached.mockResolvedValue("saved-key");
+ });
+
+ afterEach(() => {
+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = originalTableFormat;
+ });
+
+ it("neither reads nor writes for a parse request with a table format", async () => {
+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = "dynamic";
+
+ expect(await lookupAndSave(parseMeta())).toBeNull();
+ expect(getCached).not.toHaveBeenCalled();
+ expect(saveCached).not.toHaveBeenCalled();
+ });
+
+ it("keeps the cache for parse when the table format is off", async () => {
+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = "off";
+
+ expect(await lookupAndSave(parseMeta())).toMatchObject({
+ markdown: "cached",
+ });
+ expect(getCached).toHaveBeenCalled();
+ expect(saveCached).toHaveBeenCalled();
+ });
+
+ it("keeps the cache for scrape requests", async () => {
+ (config as any).PARSE_FIRE_PDF_TABLE_FORMAT = "dynamic";
+
+ expect(await lookupAndSave(makeMeta())).toMatchObject({
+ markdown: "cached",
+ });
+ expect(saveCached).toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts
new file mode 100644
index 000000000..daf150742
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/__tests__/parseFirePdfRouting.test.ts
@@ -0,0 +1,209 @@
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { detectPdf, processPdf } from "@mendable/firecrawl-rs";
+import { config } from "../../../../../config";
+import { scrapePDFWithFirePDF } from "../firePDF";
+import { scrapePDFWithParsePDF } from "../pdfParse";
+import { scrapePDF } from "../index";
+
+vi.mock("@mendable/firecrawl-rs", () => ({
+ processPdf: vi.fn(),
+ detectPdf: vi.fn(),
+}));
+
+vi.mock("../firePDF", async importOriginal => ({
+ ...(await importOriginal<typeof import("../firePDF")>()),
+ scrapePDFWithFirePDF: vi.fn(),
+}));
+
+vi.mock("../pdfParse", () => ({
+ scrapePDFWithParsePDF: vi.fn(),
+}));
+
+vi.mock("../markdownToHtml", () => ({
+ safeMarkdownToHtml: vi.fn(async (markdown: string) => `<p>${markdown}</p>`),
+}));
+
+const mockedProcessPdf = vi.mocked(processPdf);
+const mockedDetectPdf = vi.mocked(detectPdf);
+const mockedFirePdf = vi.mocked(scrapePDFWithFirePDF);
+const mockedPdfParse = vi.mocked(scrapePDFWithParsePDF);
+
+const configKeys = [
+ "PDF_RUST_EXTRACT_ENABLE",
+ "FIRE_PDF_ENABLE",
+ "FIRE_PDF_BASE_URL",
+ "FIRE_PDF_PERCENT",
+ "FIRE_PDF_ASYNC_PERCENT",
+ "MINERU_PERCENT",
+ "RUNPOD_MU_API_KEY",
+ "RUNPOD_MU_POD_ID",
+ "PARSE_PDF_ALWAYS_FIRE_PDF",
+] as const;
+const originalConfig = Object.fromEntries(
+ configKeys.map(key => [key, config[key]]),
+);
+
+function setConfig(
+ overrides: Partial<Record<(typeof configKeys)[number], unknown>>,
+) {
+ Object.assign(config as any, overrides);
+}
+
+const tempDirs: string[] = [];
+
+async function makeMeta(
+ internalOptions: Record<string, unknown>,
+ parsers: unknown[] = ["pdf"],
+) {
+ const dir = await mkdtemp(path.join(tmpdir(), "parse-fire-pdf-"));
+ tempDirs.push(dir);
+ const filePath = path.join(dir, "doc.pdf");
+ await writeFile(filePath, "%PDF-1.7\n%test\n");
+ const logger: any = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ };
+ logger.child = vi.fn(() => logger);
+ return {
+ id: "01a00000-0000-7000-8000-000000000000",
+ url: "https://parse.firecrawl.dev/uploads/doc.pdf",
+ options: { parsers },
+ logger,
+ featureFlags: new Set(["pdf"]),
+ pdfPrefetch: {
+ filePath,
+ url: "https://parse.firecrawl.dev/uploads/doc.pdf",
+ status: 200,
+ proxyUsed: "basic",
+ },
+ abort: {
+ throwIfAborted: vi.fn(),
+ asSignal: vi.fn(() => new AbortController().signal),
+ scrapeTimeout: vi.fn(() => 60_000),
+ },
+ internalOptions: { teamId: "team-test", ...internalOptions },
+ } as any;
+}
+
+const eligibleTextPdf = {
+ pdfType: "TextBased",
+ confidence: 0.99,
+ isComplex: false,
+ pageCount: 1,
+ markdown: "direct extraction",
+ title: null,
+ pagesNeedingOcr: [],
+ logs: [],
+};
+
+describe("PDF routing for /v2/parse", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ setConfig({
+ PDF_RUST_EXTRACT_ENABLE: true,
+ FIRE_PDF_ENABLE: true,
+ FIRE_PDF_BASE_URL: "http://fire-pdf.test",
+ FIRE_PDF_PERCENT: 0,
+ FIRE_PDF_ASYNC_PERCENT: 0,
+ MINERU_PERCENT: 0,
+ RUNPOD_MU_API_KEY: undefined,
+ RUNPOD_MU_POD_ID: undefined,
+ PARSE_PDF_ALWAYS_FIRE_PDF: true,
+ });
+ mockedProcessPdf.mockResolvedValue(eligibleTextPdf as any);
+ mockedDetectPdf.mockResolvedValue({
+ pdfType: "TextBased",
+ pageCount: 1,
+ title: null,
+ logs: [],
+ } as any);
+ mockedFirePdf.mockResolvedValue({
+ markdown: "fire-pdf",
+ html: "<p>fire-pdf</p>",
+ pagesProcessed: 1,
+ });
+ mockedPdfParse.mockResolvedValue({
+ markdown: "pdf-parse",
+ html: "<p>pdf-parse</p>",
+ });
+ });
+
+ afterEach(async () => {
+ await Promise.all(
+ tempDirs.splice(0).map(dir => rm(dir, { recursive: true, force: true })),
+ );
+ });
+
+ afterAll(() => {
+ Object.assign(config as any, originalConfig);
+ });
+
+ it("sends a direct-extraction-eligible parse PDF to FirePDF regardless of FIRE_PDF_PERCENT", async () => {
+ const result = await scrapePDF(await makeMeta({ isParse: true }));
+
+ expect(result.markdown).toBe("fire-pdf");
+ expect(mockedProcessPdf).not.toHaveBeenCalled();
+ expect(mockedDetectPdf).toHaveBeenCalledTimes(1);
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ });
+
+ it("keeps parse PDFs away from the MinerU diversion", async () => {
+ setConfig({ MINERU_PERCENT: 100 });
+
+ const result = await scrapePDF(await makeMeta({ isParse: true }));
+
+ expect(result.markdown).toBe("fire-pdf");
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ });
+
+ it("falls back to pdf-parse when FirePDF fails", async () => {
+ mockedFirePdf.mockRejectedValueOnce(new Error("fire-pdf 503"));
+
+ const result = await scrapePDF(await makeMeta({ isParse: true }));
+
+ expect(result.markdown).toBe("pdf-parse");
+ expect(mockedFirePdf).toHaveBeenCalledTimes(1);
+ expect(mockedPdfParse).toHaveBeenCalledTimes(1);
+ });
+
+ it("keeps direct extraction for scrape requests", async () => {
+ setConfig({ FIRE_PDF_PERCENT: 100 });
+
+ const result = await scrapePDF(await makeMeta({}));
+
+ expect(result.markdown).toBe("direct extraction");
+ expect(mockedProcessPdf).toHaveBeenCalledTimes(1);
+ expect(mockedFirePdf).not.toHaveBeenCalled();
+ });
+
+ it("keeps direct extraction for parse in fast mode", async () => {
+ const result = await scrapePDF(
+ await makeMeta({ isParse: true }, [{ type: "pdf", mode: "fast" }]),
+ );
+
+ expect(result.markdown).toBe("direct extraction");
+ expect(mockedFirePdf).not.toHaveBeenCalled();
+ });
+
+ it("restores direct extraction for parse when the flag is off", async () => {
+ setConfig({ PARSE_PDF_ALWAYS_FIRE_PDF: false });
+
+ const result = await scrapePDF(await makeMeta({ isParse: true }));
+
+ expect(result.markdown).toBe("direct extraction");
+ expect(mockedFirePdf).not.toHaveBeenCalled();
+ });
+
+ it("keeps direct extraction for parse when FirePDF is not configured", async () => {
+ setConfig({ FIRE_PDF_ENABLE: false, FIRE_PDF_BASE_URL: undefined });
+
+ const result = await scrapePDF(await makeMeta({ isParse: true }));
+
+ expect(result.markdown).toBe("direct extraction");
+ expect(mockedFirePdf).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
index 9bd140119..39b13a047 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference-flow.ts
@@ -10,6 +10,7 @@ import {
import { cacheKeyShape, tryGetCached } from "./cache";
import { lookupAdoptableFirePdfJob } from "./lookup";
import { buildFirePdfJobOptions, FirePdfAsyncFailure } from "./utils";
+import { firePdfTableFormat } from "./request-metadata";
type FirePdfByReferenceAttemptArgs = {
meta: Meta;
@@ -145,6 +146,7 @@ export async function runFirePdfByReferenceAttempt(
includePageMarkdown,
includeBlocks,
pageMarkers,
+ tableFormat: firePdfTableFormat(meta),
}),
);
meta.abort.throwIfAborted();
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
index 4014eb5d8..22b42e37c 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/by-reference.ts
@@ -13,6 +13,7 @@ import {
import { deterministicPercentage } from "./routing";
import { storage } from "../../../../../lib/gcs-jobs";
import { FIRE_PDF_BY_REFERENCE_MAX_FILE_SIZE } from "../types";
+import { parseAlwaysFirePdf } from "./request-metadata";
/** Input handle for a FirePDF async submit that travels by GCS reference
* instead of inline base64. Produced by {@link uploadPdfInputForFirePdf}. */
@@ -199,10 +200,12 @@ export function largePdfLimitBytes(meta: Meta): number {
* id (same distribution as a random draw) precisely so every gate — the
* fire-engine handoff grant, the download admission, and the PDF engine's
* routing — sees the SAME verdict; a per-call random draw would let them
- * drift. Keyed apart from the async-cohort hash. */
+ * drift. Keyed apart from the async-cohort hash. Parse requests that must
+ * reach FirePDF (parseAlwaysFirePdf) are never diverted. */
export function mineruDiverted(meta: Meta): boolean {
return (
config.MINERU_PERCENT > 0 &&
+ !parseAlwaysFirePdf(meta) &&
deterministicPercentage(`mineru:${meta.id}`) < config.MINERU_PERCENT
);
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
index 170b2daed..af8976c9d 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/cache.ts
@@ -20,6 +20,7 @@ import {
firePdfCacheRefusedWritesTotal,
} from "./metrics";
import { cacheServiceConfigured, lookupCachedResult } from "./cache-service";
+import { firePdfTableFormat } from "./request-metadata";
import { consumeRefresh, refreshDecisionFor } from "./refresh-budget";
import {
firePdfBlockPagesSchema,
@@ -191,6 +192,9 @@ export async function tryGetCached(
pageMarkers = false,
): Promise<PDFProcessorResult | null> {
if (meta.internalOptions.zeroDataRetention) return null;
+ // Cache entries hold FirePDF's default table rendering; FirePDF itself
+ // neither reads nor writes the cache for an explicit tableFormat.
+ if (firePdfTableFormat(meta) !== undefined) return null;
const viaService = cacheServiceConfigured();
// No service and no bucket, no cache: nothing to hash, no refresh budget to spend.
if (!viaService && !pdfCacheConfigured()) return null;
@@ -412,6 +416,7 @@ export async function maybeSaveResult(args: {
failedPages,
} = args;
if (meta.internalOptions.zeroDataRetention) return;
+ if (firePdfTableFormat(meta) !== undefined) return;
// With the lookup service in use, fire-pdf writes the result itself.
if (cacheServiceConfigured()) return;
if (!pdfCacheConfigured()) return;
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
index 07b909ee6..ea46723f9 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/request-metadata.ts
@@ -1,4 +1,6 @@
import type { Meta } from "../../..";
+import { config } from "../../../../../config";
+import { getPDFMode } from "../../../../../controllers/v2/types";
import { hasCustomRequestContext } from "../../../lib/request-context";
/** What we hand fire-pdf: a PDF, or a raster image wrapped as a one-page document. */
@@ -12,6 +14,29 @@ export function firePdfRequestKind(meta: Meta): "scrape" | "parse" {
: "scrape";
}
+/** Whether this PDF must go to FirePDF instead of direct extraction, the
+ * FIRE_PDF_PERCENT sample, or the MinerU diversion (see
+ * PARSE_PDF_ALWAYS_FIRE_PDF). Fast mode keeps its text-layer-only contract. */
+export function parseAlwaysFirePdf(meta: Meta): boolean {
+ return (
+ firePdfRequestKind(meta) === "parse" &&
+ config.PARSE_PDF_ALWAYS_FIRE_PDF &&
+ getPDFMode(meta.options.parsers) !== "fast" &&
+ !!config.FIRE_PDF_ENABLE &&
+ !!config.FIRE_PDF_BASE_URL
+ );
+}
+
+/** The `tableFormat` sent to FirePDF, or undefined for its default (pipe
+ * tables). Only parse requests set one (PARSE_FIRE_PDF_TABLE_FORMAT). */
+export function firePdfTableFormat(
+ meta: Meta,
+): "markdown" | "html" | "dynamic" | undefined {
+ if (firePdfRequestKind(meta) !== "parse") return undefined;
+ const format = config.PARSE_FIRE_PDF_TABLE_FORMAT;
+ return format === "off" ? undefined : format;
+}
+
export function buildFirePdfRequestMetadata(
meta: Meta,
sourceKind: FirePdfSourceKind = "pdf",
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
index 38a924820..540472b50 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/submit.ts
@@ -2,7 +2,10 @@ import type { Meta } from "../../..";
import type { PDFMode } from "../../../../../controllers/v2/types";
import { fetch as undiciFetch } from "undici";
import { AbortManagerThrownError } from "../../../lib/abortManager";
-import { buildFirePdfRequestMetadata } from "./request-metadata";
+import {
+ buildFirePdfRequestMetadata,
+ firePdfTableFormat,
+} from "./request-metadata";
import {
firePdfAsyncSubmit503Total,
firePdfAsyncSubmitRetriesTotal,
@@ -157,6 +160,7 @@ export async function submitJob(args: SubmitArgs): Promise<SubmitOutcome> {
includePageMarkdown,
includeBlocks,
pageMarkers,
+ tableFormat: firePdfTableFormat(meta),
}),
};
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
index 6e7e06027..82b0c67f7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/fire-pdf/utils.ts
@@ -158,6 +158,7 @@ export function buildFirePdfJobOptions(args: {
includePageMarkdown: boolean;
includeBlocks: boolean;
pageMarkers: boolean;
+ tableFormat: string | undefined;
}): Record<string, unknown> {
return {
// fire-pdf accepts only a positive integer here. When the router has no
@@ -179,6 +180,7 @@ export function buildFirePdfJobOptions(args: {
// `page_markers`. Sending snake_case here would be rejected as an
// unknown option.
...(args.pageMarkers && { pageMarkers: true }),
+ ...(args.tableFormat !== undefined && { tableFormat: args.tableFormat }),
};
}
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
index dd2ff5aee..dd6e7015a 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/firePDF.ts
@@ -14,6 +14,7 @@ import {
import { firePdfBlocksSchema, firePdfPagesSchema } from "./fire-pdf/schema";
import {
buildFirePdfRequestMetadata,
+ firePdfTableFormat,
type FirePdfSourceKind,
} from "./fire-pdf/request-metadata";
@@ -106,6 +107,7 @@ export async function scrapePDFWithFirePDF(
const zdr = meta.internalOptions.zeroDataRetention === true;
const pdfSha256 = createPdfCacheKey(base64Content);
+ const tableFormat = firePdfTableFormat(meta);
// Explicit deadline contract with fire-pdf (mirrors mineru-api):
// timeout — remaining scrape-tier budget in ms (from AbortManager)
@@ -141,6 +143,7 @@ export async function scrapePDFWithFirePDF(
...(includePageMarkdown && { include_page_markdown: true }),
...(includeBlocks && { include_blocks: true }),
...(pageMarkers && { page_markers: true }),
+ ...(tableFormat && { tableFormat }),
// Enrichment for the fire-pdf jobs DB / dashboard. fire-pdf treats
// these as optional — older fire-pdf builds will ignore unknown fields.
team_id: meta.internalOptions.teamId,
diff --git a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
index b43c37a94..138218220 100644
--- a/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/pdf/index.ts
@@ -60,6 +60,7 @@ import {
firePdfFeaturesLabel,
recordFirePdfRoute,
} from "./fire-pdf/routing";
+import { parseAlwaysFirePdf } from "./fire-pdf/request-metadata";
import { scrapePDFWithParsePDF } from "./pdfParse";
import { toPublicBlocks } from "./blocks";
import { isPdfBuffer, PDF_SNIFF_WINDOW } from "./pdfUtils";
@@ -237,6 +238,10 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
pageMarkers) &&
!!config.FIRE_PDF_BASE_URL;
+ // /v2/parse: every non-fast PDF goes to FirePDF (PARSE_PDF_ALWAYS_FIRE_PDF).
+ // Unlike forceFirePDF this keeps the MinerU / pdf-parse fallbacks.
+ const alwaysFirePdf = parseAlwaysFirePdf(meta);
+
// The MinerU diversion is deterministic on the scrape id (see
// mineruDiverted) so this routing verdict is BY CONSTRUCTION the same
// one inside byReferenceReachableForRequest() — and the same one
@@ -329,7 +334,13 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
});
}
- if (!rustEnabled || mode === "ocr" || forceFirePDF || routeToMinerU) {
+ if (
+ !rustEnabled ||
+ mode === "ocr" ||
+ forceFirePDF ||
+ routeToMinerU ||
+ alwaysFirePdf
+ ) {
// Legacy / OCR path: detect metadata only, skip Rust extraction.
// When PDF_RUST_EXTRACT_ENABLE is off this is the only path taken,
// matching current prod behaviour (detectPdf → MinerU → pdfParse).
@@ -360,6 +371,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
url: meta.rewrittenUrl ?? meta.url,
rustEnabled,
mode,
+ alwaysFirePdf,
});
totalPageCount = detection.pageCount;
@@ -657,7 +669,7 @@ export async function scrapePDF(meta: Meta): Promise<EngineScrapeResult> {
config.FIRE_PDF_ENABLE &&
config.FIRE_PDF_BASE_URL &&
fileSizeBytes < FIRE_PDF_MAX_FILE_SIZE &&
- Math.random() * 100 < config.FIRE_PDF_PERCENT));
+ (alwaysFirePdf || Math.random() * 100 < config.FIRE_PDF_PERCENT)));
if (useFirePDF) {
// Async is a server-controlled cohort within traffic already selected