Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): hosted bash for agents at /v2/bash (experiment, off by default) (#5664)

Benjamin Werner793fd57f133b
diff --git a/apps/api/package.json b/apps/api/package.json
index 9a8757cbf..9f545f41b 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -127,6 +127,7 @@
"jose": "^6.2.12",
"jsdom": "^29.1.1",
"jsonrepair": "^3.14.0",
+ "just-bash": "3.6.0",
"koffi": "^2.9.0",
"langsmith": "^0.6.0",
"lodash": "^4.18.0",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index cfb628b13..12b5d6045 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -187,7 +187,7 @@ importers:
version: 16.4.5
drizzle-orm:
specifier: 1.0.0-rc.3
- version: 1.0.0-rc.3(@opentelemetry/[email protected])(@types/[email protected])([email protected])([email protected])
esbuild:
specifier: ^0.28.1
version: 0.28.1
@@ -230,6 +230,9 @@ importers:
jsonrepair:
specifier: ^3.14.0
version: 3.14.0
+ just-bash:
+ specifier: 3.6.0
+ version: 3.6.0
koffi:
specifier: ^2.9.0
version: 2.9.0
@@ -398,7 +401,7 @@ importers:
version: 1.5.0
'@napi-rs/cli':
specifier: ^3.7.3
- version: 3.7.3(@emnapi/[email protected])(@emnapi/[email protected])(@types/[email protected])
'@taplo/cli':
specifier: ^0.7.0
version: 0.7.0
@@ -554,6 +557,9 @@ packages:
'@asamuzakjp/[email protected]':
resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}
+ '@borewit/[email protected]':
+ resolution: {integrity: sha512-49CycTz1aAZmvFcwkeFyhud1HUmRIB5Tabf7ALV+VA4Iu85MTfz2cUwzPagMFAdVBvrWdkCxG67I30tUwwpKsw==}
+
'@bramus/[email protected]':
resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==}
hasBin: true
@@ -1046,6 +1052,10 @@ packages:
'@mixmark-io/[email protected]':
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
+ '@mongodb-js/[email protected]':
+ resolution: {integrity: sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==}
+ engines: {node: '>= 20.19.0'}
+
'@msgpackr-extract/[email protected]':
resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==}
cpu: [arm64]
@@ -2018,6 +2028,13 @@ packages:
resolution: {integrity: sha512-Ck3zFhQhIhi02Hl6T4ZmJsXdnJE+wXcJz5f8klxd4keRYgenMnip3JDPMGDRLbnC/2iGd8P0sBIQqI3KxfVjBg==}
hasBin: true
+ '@tokenizer/[email protected]':
+ resolution: {integrity: sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==}
+ engines: {node: '>=18'}
+
+ '@tokenizer/[email protected]':
+ resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}
+
'@tootallnate/[email protected]':
resolution: {integrity: sha512-VyMVKRrpHTT8PnotUeV8L/mDaMwD5DaAKCFLP73zAqAtvF0FCqky+Ki7BYbFCYQmqFyTe9316Ed5zS70QUR9eg==}
engines: {node: '>= 10'}
@@ -2449,6 +2466,9 @@ packages:
resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}
+ resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}
+
resolution: {integrity: sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==}
engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}
@@ -2473,6 +2493,9 @@ packages:
resolution: {integrity: sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==}
+ resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
+
resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}
@@ -2513,6 +2536,9 @@ packages:
resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}
engines: {node: '>= 6'}
+ resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
+
resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}
engines: {node: '>=18'}
@@ -2571,6 +2597,10 @@ packages:
resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}
engines: {node: '>=20'}
+ resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}
@@ -2687,6 +2717,14 @@ packages:
resolution: {integrity: sha512-vWHQkXgg2cE3BB4rR0c+PYw9BCQzLoNIRMI7AunIkZst74tw/vpPp7VLoYtsR/cy1f55g3XDWUslfeCZjiZJyA==}
engines: {node: '>=14.16'}
+ resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
+ engines: {node: '>=10'}
+
+ resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}
+ engines: {node: '>=4.0.0'}
+
resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}
engines: {node: '>=0.4.0'}
@@ -2974,6 +3012,10 @@ packages:
resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==}
engines: {node: '>=18.0.0'}
+ resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}
+ engines: {node: '>=6'}
+
resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==}
engines: {node: '>=12.0.0'}
@@ -3044,6 +3086,10 @@ packages:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
engines: {node: ^12.20 || >= 14.13}
+ resolution: {integrity: sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==}
+ engines: {node: '>=20'}
+
resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}
@@ -3102,6 +3148,9 @@ packages:
resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}
+ resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}
+
resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -3164,6 +3213,9 @@ packages:
resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==}
+ resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}
+
resolution: {integrity: sha512-A1vjIwC3qpdCso45Iw127gnp7sPhqgY8lUQhDrArIs+GbmfaaVPFuoyy7WyLT9EPZCRZ7j7aLdwUhQmHWesEow==}
engines: {node: 18 || 20 || >=22}
@@ -3220,6 +3272,10 @@ packages:
resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-PSaNyJk8VJlUVEhMHPXT/Z/WgNrLBh43WaYQJiu1qvyCuPQA+kgFMNmAOxI3GU3uXrCkO/xRW11XjcswrksYOA==}
+ engines: {node: '>=20.19'}
+
resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}
engines: {node: '>= 0.4'}
@@ -3303,6 +3359,13 @@ packages:
resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}
+ resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}
+
+ resolution: {integrity: sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==}
+ engines: {node: ^20.17.0 || >=22.9.0}
+
resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}
engines: {node: '>=12.22.0'}
@@ -3407,6 +3470,11 @@ packages:
resolution: {integrity: sha512-tWPGKMZf/8UPim+fcW2EfcQ/d/7aKUrP6IECz9G3Tu6Q5dX0orSleqJ9z6sSw7qrQkjF8/Edo4DvsWBZ8H+HNg==}
hasBin: true
+ resolution: {integrity: sha512-2COjEgcHdqKXeQSiNfPAHmpwQx4S8wATaIwog83g2t645U5tom6FED4CBBdr3M+0ioX4xZ26eODw3/nD6RQ6xw==}
+ engines: {node: '>=20.19'}
+ hasBin: true
+
resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==}
@@ -3636,10 +3704,17 @@ packages:
resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
+ engines: {node: '>=10'}
+
resolution: {integrity: sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==}
engines: {node: 18 || 20 || >=22}
+ resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
+
resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}
engines: {node: '>=16 || 14 >=14.17'}
@@ -3648,6 +3723,13 @@ packages:
resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==}
engines: {node: '>=16 || 14 >=14.17'}
+ resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}
+
+ resolution: {integrity: sha512-t9VmxaqrmANnEOBhpSDI6HD192Ge48k8vmWqQQL7hSFEqHEYwZbbsu49+aKLWZeRvFs3j1pMhXOqqF4kPlvjkQ==}
+ engines: {node: '>=18.0.0'}
+
resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}
@@ -3674,6 +3756,9 @@ packages:
engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}
hasBin: true
+ resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==}
+
resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==}
engines: {node: '>= 0.6'}
@@ -3682,6 +3767,10 @@ packages:
resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==}
engines: {node: '>= 0.6'}
+ resolution: {integrity: sha512-bkb/RkuWCE64eWu1PCh3Fu2pGc9mKRs+kOE+FU2bGHRYlQa86/D321DSATV54AsCQMKz3/xSt9sRgraZTpqdhQ==}
+ engines: {node: '>=10'}
+
resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}
deprecated: This package is no longer needed on supported Node.js versions. Node.js 16+ includes AbortController and AbortSignal as built-in globals. All Node.js versions requiring this polyfill are end-of-life. Use the built-in globals instead.
@@ -3689,6 +3778,10 @@ packages:
resolution: {integrity: sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==}
+ resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==}
+ engines: {node: ^18 || ^20 || >= 21}
+
resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}
@@ -3721,6 +3814,11 @@ packages:
resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==}
hasBin: true
+ resolution: {integrity: sha512-s0KzNOWwOJJgPG6wxg6cKohnAl9Wk/oW1KrQaVzJBjQwVcUGPQCzpR46Ximygjqj/3KhOrtJXnYMp/xYAXp75g==}
+ engines: {node: '>=16.0.0'}
+ hasBin: true
+
resolution: {integrity: sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==}
engines: {node: ^18.17.0 || >=20.5.0}
@@ -3822,6 +3920,9 @@ packages:
resolution: {integrity: sha512-zAMfbta9nPA8AKkA4WdmNAT8R8s6Dk0/xwPsqNnRidMFcLo1YkLuPDTgs2f64XdzS8vbLNnoJ+uUIkgbz7oRHA==}
+ resolution: {integrity: sha512-qOR+UDWv6D3hMxnWWOk0ZpBdCkd+kw5Yh7aXOaNSQU8A+73EA2bhW7EiUix/hY0yulUi5FRvt7PpYSI2VDvY0g==}
+
resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}
@@ -3943,6 +4044,12 @@ packages:
resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==}
+ engines: {node: '>=10'}
+ deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available.
+ hasBin: true
+
resolution: {integrity: sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==}
engines: {node: '>=14'}
@@ -4017,6 +4124,13 @@ packages:
resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==}
engines: {node: '>= 0.10'}
+ resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}
+ hasBin: true
+
+ resolution: {integrity: sha512-s/I5zEAo79SUK0Qw4dpZKpiMwbQ6Gz0KU2NRr7eaO4x/p2g7Vvmn3hdeXDg8VsaUjfj/ora+e9oi27LX/C9+mw==}
+
resolution: {integrity: sha512-qpt8EwugBWDw2cgE2W+/3oxC+KTez2uSVR8JU9Q36TXPAGCaozfQUs59v4j4GFpWTaw0i6hAZSvOmu1J0uOEUg==}
engines: {node: ^18.17.0 || >=20.5.0}
@@ -4110,6 +4224,15 @@ packages:
resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==}
engines: {node: '>= 18'}
+ resolution: {integrity: sha512-KUurW05zgjtZ2su0cBURC1q2RG/Uq0uzV3qURDsXqiKh03p9IXABQK4Qem56cFMqjT8TdQ2iTWsePrMprQWIRA==}
+ engines: {node: '>=20.19.0'}
+ peerDependencies:
+ typescript: '>=5.1.6 <7'
+ peerDependenciesMeta:
+ typescript:
+ optional: true
+
resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}
@@ -4127,6 +4250,10 @@ packages:
resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==}
engines: {node: '>=v12.22.7'}
+ resolution: {integrity: sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg==}
+ hasBin: true
+
resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==}
engines: {node: '>=10'}
@@ -4183,6 +4310,12 @@ packages:
resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}
engines: {node: '>=14'}
+ resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}
+
+ resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}
+
resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}
@@ -4213,6 +4346,12 @@ packages:
resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==}
+ resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==}
+
+ resolution: {integrity: sha512-q/JOZuJfmVh8Zf8Eo+cMJvtxAWeWBqTfJ6AQYGXZmcaLGHMwhA2niH3fQYHrBtPeOss1ttzHec07K5+rHaMRMw==}
+
resolution: {integrity: sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==}
engines: {node: '>=4'}
@@ -4284,6 +4423,10 @@ packages:
resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==}
engines: {node: '>=12'}
+ resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}
+ engines: {node: '>=0.10.0'}
+
resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==}
engines: {node: '>=14.16'}
@@ -4295,6 +4438,10 @@ packages:
resolution: {integrity: sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA==}
+ resolution: {integrity: sha512-iAqrCpp7UXD2FZiHWhLXqdJ6CmNeVzRkkHAyyhhKMVtbVm2KtPX3D/bJNonvtcyLTHS4rwqP2pz9fsjK1K4z0Q==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==}
@@ -4317,6 +4464,13 @@ packages:
os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android]
hasBin: true
+ resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==}
+
+ resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}
+ engines: {node: '>=6'}
+
resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}
@@ -4367,6 +4521,10 @@ packages:
resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}
engines: {node: '>=0.6'}
+ resolution: {integrity: sha512-s9euYHM//T3loJtbNUYdbAzsMlvKBeRS+XCLFy+UGosMQH19Hwjap5k19i4PPAEJJBsVIpMpVTxl4fIqCuNfUg==}
+ engines: {node: '>=14.16'}
+
resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==}
engines: {node: '>=6'}
@@ -4404,9 +4562,16 @@ packages:
engines: {node: '>=18.0.0'}
hasBin: true
+ resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}
+
resolution: {integrity: sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==}
+ resolution: {integrity: sha512-EKZvCCFHh8sWttpAMs10U6niZFJ2cpz8LkCPlylxoG4JcHF50J7EHnnBW3AY1qXtUMu7kDjCp7dnggB1sIbqEQ==}
+ engines: {node: '>=18', npm: '>=9'}
+
resolution: {integrity: sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==}
@@ -4426,6 +4591,11 @@ packages:
engines: {node: '>=14.17'}
hasBin: true
+ resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}
+ engines: {node: '>=14.17'}
+ hasBin: true
+
resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==}
engines: {node: '>=14.17'}
@@ -4436,6 +4606,10 @@ packages:
engines: {node: '>=16.20.0'}
hasBin: true
+ resolution: {integrity: sha512-8iAasVS4wUx0gPLjH8Xtz2PeDzTSiy8QiLGu2DT3X5GU+egFEQhpnbtkehbAwjvDcxIERmCYcysiODFmE3Ud0Q==}
+ engines: {node: '>=18'}
+
resolution: {integrity: sha512-PnKvRX+Fbxl/usVlTr/31tXIOVAH+8OdC9qaxUQcvurwEnoc3Vk/ipi2Ce8IjMxzK9byfSGdsUBjqqJGc1o5lw==}
engines: {node: '>=14'}
@@ -4873,6 +5047,8 @@ snapshots:
'@asamuzakjp/[email protected]': {}
+ '@borewit/[email protected]': {}
+
'@bramus/[email protected]':
dependencies:
css-tree: 3.2.1
@@ -5337,6 +5513,12 @@ snapshots:
'@mixmark-io/[email protected]': {}
+ '@mongodb-js/[email protected]':
+ dependencies:
+ node-addon-api: 8.9.2
+ prebuild-install: 7.1.3
+ optional: true
+
'@msgpackr-extract/[email protected]':
optional: true
@@ -5355,7 +5537,7 @@ snapshots:
'@msgpackr-extract/[email protected]':
optional: true
dependencies:
'@inquirer/prompts': 8.5.2(@types/[email protected])
'@napi-rs/cross-toolchain': 1.0.3(@emnapi/[email protected])(@emnapi/[email protected])
@@ -5363,7 +5545,7 @@ snapshots:
'@octokit/rest': 22.0.1
clipanion: 4.0.0-rc.4([email protected])
colorette: 2.0.20
- emnapi: 1.11.2
+ emnapi: 1.11.2([email protected])
es-toolkit: 1.49.0
js-yaml: 4.3.2
obug: 2.1.4
@@ -6038,6 +6220,15 @@ snapshots:
'@taplo/[email protected]': {}
+ '@tokenizer/[email protected]':
+ dependencies:
+ debug: 4.4.3
+ token-types: 6.2.0
+ transitivePeerDependencies:
+ - supports-color
+
+ '@tokenizer/[email protected]': {}
+
'@tootallnate/[email protected]': {}
@@ -6418,6 +6609,13 @@ snapshots:
+ dependencies:
+ buffer: 5.7.1
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ optional: true
+
dependencies:
bytes: 3.1.2
@@ -6461,6 +6659,12 @@ snapshots:
+ dependencies:
+ base64-js: 1.5.1
+ ieee754: 1.2.1
+ optional: true
+
dependencies:
base64-js: 1.5.1
@@ -6519,6 +6723,9 @@ snapshots:
parse5: 7.1.2
parse5-htmlparser2-tree-adapter: 7.0.0
+ optional: true
+
dependencies:
restore-cursor: 5.1.0
@@ -6577,6 +6784,8 @@ snapshots:
+
@@ -6678,6 +6887,14 @@ snapshots:
+ dependencies:
+ mimic-response: 3.1.0
+ optional: true
+
+ optional: true
+
@@ -6719,11 +6936,12 @@ snapshots:
optionalDependencies:
'@opentelemetry/api': 1.9.1
'@types/pg': 8.15.5
pg: 8.16.3
+ sql.js: 1.16.0
zod: 4.6.7
@@ -6749,7 +6967,9 @@ snapshots:
+ optionalDependencies:
+ node-addon-api: 8.9.2
@@ -6849,6 +7069,9 @@ snapshots:
+ optional: true
+
@@ -6947,6 +7170,15 @@ snapshots:
node-domexception: 1.0.0
web-streams-polyfill: 3.3.3
+ dependencies:
+ '@tokenizer/inflate': 0.4.1
+ strtok3: 10.7.0
+ token-types: 6.2.0
+ uint8array-extras: 1.6.0
+ transitivePeerDependencies:
+ - supports-color
+
optional: true
@@ -7011,6 +7243,9 @@ snapshots:
+ optional: true
+
optional: true
@@ -7108,6 +7343,9 @@ snapshots:
dependencies:
resolve-pkg-maps: 1.0.0
+ optional: true
+
dependencies:
minimatch: 11.0.3
@@ -7211,6 +7449,11 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ dependencies:
+ ipaddr.js: 2.2.0
+ undici: 7.30.1
+
@@ -7306,6 +7549,11 @@ snapshots:
+ optional: true
+
+
dependencies:
'@ioredis/commands': 1.2.0
@@ -7404,6 +7652,30 @@ snapshots:
+ dependencies:
+ diff: 8.0.3
+ fast-xml-parser: 5.12.0
+ file-type: 21.3.4
+ guarded-fetch: 0.1.3
+ ini: 6.0.0
+ modern-tar: 0.7.7
+ papaparse: 5.7.1
+ re2js: 1.3.3
+ run: 2.1.3([email protected])
+ seek-bzip: 2.0.0
+ smol-toml: 1.9.0
+ sprintf-js: 1.1.3
+ sql.js: 1.16.0
+ turndown: 7.2.5
+ typescript: 5.9.3
+ yaml: 2.9.1
+ optionalDependencies:
+ '@mongodb-js/zstd': 7.0.0
+ node-liblzma: 2.2.0
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
buffer-equal-constant-time: 1.0.1
@@ -7595,15 +7867,26 @@ snapshots:
+ optional: true
+
dependencies:
'@isaacs/nonrecursive-glob-match': 1.0.3
brace-expansion: 5.0.12
+ optional: true
+
+ optional: true
+
+
@@ -7634,15 +7917,26 @@ snapshots:
+ optional: true
+
+ dependencies:
+ semver: 7.8.5
+ optional: true
+
optional: true
+ optional: true
+
@@ -7666,6 +7960,12 @@ snapshots:
+ dependencies:
+ node-addon-api: 8.9.2
+ node-gyp-build: 4.8.4
+ optional: true
+
@@ -7799,6 +8099,8 @@ snapshots:
+
@@ -7905,6 +8207,22 @@ snapshots:
dependencies:
xtend: 4.0.2
+ dependencies:
+ detect-libc: 2.1.2
+ expand-template: 2.0.3
+ github-from-package: 0.0.0
+ minimist: 1.2.8
+ mkdirp-classic: 0.5.3
+ napi-build-utils: 2.0.0
+ node-abi: 3.101.0
+ pump: 3.0.4
+ rc: 1.2.8
+ simple-get: 4.0.1
+ tar-fs: 2.1.5
+ tunnel-agent: 0.6.0
+ optional: true
+
@@ -7991,6 +8309,16 @@ snapshots:
iconv-lite: 0.7.3
unpipe: 1.0.0
+ dependencies:
+ deep-extend: 0.6.0
+ ini: 1.3.8
+ minimist: 1.2.8
+ strip-json-comments: 2.0.1
+ optional: true
+
+
dependencies:
json-parse-even-better-errors: 4.0.0
@@ -8106,6 +8434,10 @@ snapshots:
transitivePeerDependencies:
- supports-color
+ optionalDependencies:
+ typescript: 5.9.3
+
@@ -8118,6 +8450,10 @@ snapshots:
dependencies:
xmlchars: 2.2.0
+ dependencies:
+ commander: 6.2.1
+
@@ -8189,6 +8525,16 @@ snapshots:
+ optional: true
+
+ dependencies:
+ decompress-response: 6.0.0
+ once: 1.4.0
+ simple-concat: 1.0.1
+ optional: true
+
dependencies:
is-arrayish: 0.3.2
@@ -8215,6 +8561,10 @@ snapshots:
dependencies:
through: 2.3.8
+
+
dependencies:
bindings: 1.5.0
@@ -8285,6 +8635,9 @@ snapshots:
dependencies:
ansi-regex: 6.4.0
+ optional: true
+
@@ -8296,6 +8649,10 @@ snapshots:
dependencies:
anynum: 1.0.1
+ dependencies:
+ '@tokenizer/token': 0.3.0
+
@@ -8324,6 +8681,23 @@ snapshots:
+ dependencies:
+ chownr: 1.1.4
+ mkdirp-classic: 0.5.3
+ pump: 3.0.4
+ tar-stream: 2.2.0
+ optional: true
+
+ dependencies:
+ bl: 4.1.0
+ end-of-stream: 1.4.5
+ fs-constants: 1.0.0
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ optional: true
+
dependencies:
bintrees: 1.0.2
@@ -8377,6 +8751,12 @@ snapshots:
+ dependencies:
+ '@borewit/text-codec': 0.2.3
+ '@tokenizer/token': 0.3.0
+ ieee754: 1.2.1
+
dependencies:
psl: 1.15.0
@@ -8415,10 +8795,19 @@ snapshots:
optionalDependencies:
fsevents: 2.3.3
+ dependencies:
+ safe-buffer: 5.2.1
+ optional: true
+
dependencies:
'@mixmark-io/domino': 2.2.0
+ dependencies:
+ '@mixmark-io/domino': 2.2.0
+
@@ -8436,6 +8825,8 @@ snapshots:
+
@@ -8461,6 +8852,8 @@ snapshots:
'@typescript/typescript-win32-arm64': 7.0.2
'@typescript/typescript-win32-x64': 7.0.2
+
diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yaml
index 112612a2a..f01831083 100644
--- a/apps/api/pnpm-workspace.yaml
+++ b/apps/api/pnpm-workspace.yaml
@@ -122,6 +122,10 @@ allowBuilds:
# @google-cloud/bigtable); the pure-JS fallback is fine for our payload
# sizes, so no native compilation.
sse4_crc32: false
+ # Optional native compressors for just-bash's archive commands, which the
+ # hosted shell (/v2/bash) does not expose.
+ "@mongodb-js/zstd": false
+ node-liblzma: false
bufferutil: true
esbuild: true
foundationdb: true
diff --git a/apps/api/src/__tests__/snips/v2/bash.test.ts b/apps/api/src/__tests__/snips/v2/bash.test.ts
new file mode 100644
index 000000000..8a48e32b7
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/bash.test.ts
@@ -0,0 +1,53 @@
+import { ALLOW_TEST_SUITE_WEBSITE, itIf, TEST_SUITE_WEBSITE } from "../lib";
+import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";
+
+describe("Hosted bash", () => {
+ let identity: Identity;
+
+ beforeAll(async () => {
+ identity = await idmux({ name: "bash", concurrency: 10, credits: 1000 });
+ }, 10000);
+
+ const run = (command: string) =>
+ request(TEST_API_URL)
+ .post("/v2/bash")
+ .set("Authorization", `Bearer ${identity.apiKey}`)
+ .type("form")
+ .send(command);
+
+ it("serves usage", async () => {
+ const res = await request(TEST_API_URL).get("/v2/bash");
+ expect(res.status).toBe(200);
+ expect(res.text).toContain("/v2/bash --data-binary 'firecrawl search");
+ });
+
+ it("runs a command in a fresh shell", async () => {
+ const res = await run("echo hello | tr a-z A-Z; cat /README.md | head -1");
+ expect(res.status).toBe(200);
+ expect(res.headers["x-exit-code"]).toBe("0");
+ expect(res.text).toBe("HELLO\n# Firecrawl bash\n");
+ });
+
+ itIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "scrapes through the API's own scrape route",
+ async () => {
+ const res = await run(
+ `firecrawl scrape ${TEST_SUITE_WEBSITE} | grep -c Firecrawl`,
+ );
+ expect(res.headers["x-exit-code"]).toBe("0");
+ expect(Number(res.text.trim())).toBeGreaterThan(0);
+ },
+ scrapeTimeout,
+ );
+
+ it("exits 2 for a command outside the hosted subset", async () => {
+ const res = await run("firecrawl crawl https://example.com");
+ expect(res.headers["x-exit-code"]).toBe("2");
+ expect(res.text).toContain("npx -y firecrawl-cli@latest init");
+ });
+
+ it("stops a runaway command", async () => {
+ const res = await run("while true; do :; done");
+ expect(res.headers["x-exit-code"]).toBe("126");
+ });
+});
diff --git a/apps/api/src/controllers/v2/__tests__/bash.test.ts b/apps/api/src/controllers/v2/__tests__/bash.test.ts
new file mode 100644
index 000000000..bff86375d
--- /dev/null
+++ b/apps/api/src/controllers/v2/__tests__/bash.test.ts
@@ -0,0 +1,348 @@
+import express from "express";
+import type { Server } from "http";
+import type { AddressInfo } from "net";
+import request from "supertest";
+import { registerHostedBashRoute } from "../bash";
+
+const DOCS_INDEX = `# Firecrawl Docs
+- [Intro](https://docs.firecrawl.dev/introduction.md): Start here.
+- [CLI](https://docs.firecrawl.dev/sdks/cli.md): The CLI.
+`;
+
+interface InnerCall {
+ path: string;
+ ip: string | undefined;
+ authorization: string | undefined;
+ body: any;
+}
+
+/**
+ * The route under test plus stand-ins for /v2/search and /v2/scrape on the same
+ * server, since the shell calls back into the server that answered it.
+ */
+describe("hosted bash route", () => {
+ const realFetch = globalThis.fetch;
+ const calls: InnerCall[] = [];
+ let respond: (call: InnerCall) => { status: number; json: unknown };
+ let server: Server;
+ let base: string;
+
+ beforeAll(async () => {
+ const app = express();
+ // One trusted hop, like the load balancer in front of the real API.
+ app.set("trust proxy", 1);
+ registerHostedBashRoute(app);
+ app.use(express.json());
+ app.post(["/v2/search", "/v2/scrape"], (req, res) => {
+ const call: InnerCall = {
+ path: req.path,
+ ip: req.ip,
+ authorization: req.get("authorization"),
+ body: req.body,
+ };
+ calls.push(call);
+ const { status, json } = respond(call);
+ res.status(status).json(json);
+ });
+ server = app.listen(0, "127.0.0.1");
+ await new Promise(resolve => server.once("listening", resolve));
+ base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+
+ // The docs mount reads docs.firecrawl.dev; everything else is the real fetch.
+ vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
+ const url = String(input);
+ if (!url.startsWith("https://docs.firecrawl.dev/")) {
+ return realFetch(input, init);
+ }
+ const body = url.endsWith("/llms.txt") ? DOCS_INDEX : `page ${url}\n`;
+ return Promise.resolve(new Response(body));
+ });
+ });
+
+ afterAll(async () => {
+ vi.restoreAllMocks();
+ await new Promise(resolve => server.close(resolve));
+ });
+
+ beforeEach(() => {
+ calls.length = 0;
+ respond = call =>
+ call.path === "/v2/scrape"
+ ? {
+ status: 200,
+ json: {
+ success: true,
+ data: {
+ markdown: "# Example",
+ links: ["https://a.test", "https://b.test"],
+ metadata: { title: "Example" },
+ },
+ },
+ }
+ : {
+ status: 200,
+ json: {
+ success: true,
+ id: "search-1",
+ creditsUsed: 2,
+ data: {
+ web: [
+ {
+ title: "First",
+ url: "https://first.test",
+ description: "About first",
+ },
+ { url: "https://second.test", markdown: "line 1\nline 2" },
+ ],
+ },
+ },
+ };
+ });
+
+ // The caller's address as the load balancer would report it.
+ const run = (command: string, ip = "203.0.113.9") =>
+ request(base)
+ .post("/v2/bash")
+ .set("X-Forwarded-For", ip)
+ .type("form")
+ .send(command);
+
+ it("serves usage that points at its own URL", async () => {
+ const res = await request(base).get("/v2/bash");
+ expect(res.status).toBe(200);
+ expect(res.text).toContain(
+ `curl ${base}/v2/bash --data-binary 'firecrawl search`,
+ );
+ });
+
+ it("runs a raw command sent the way curl --data-binary sends it", async () => {
+ const res = await run("echo 'a=1&b=2' | tr a-z A-Z; ls /");
+ expect(res.status).toBe(200);
+ expect(res.headers["x-exit-code"]).toBe("0");
+ expect(res.text).toBe("A=1&B=2\nREADME.md\nbin\ndev\ndocs\nproc\nusr\n");
+ });
+
+ it("takes a command from the query string or a JSON body, and returns JSON", async () => {
+ const get = await request(base)
+ .get("/v2/bash")
+ .query({ command: "echo from-query" });
+ expect(get.text).toBe("from-query\n");
+
+ const post = await request(base)
+ .post("/v2/bash?format=json")
+ .send({ command: "echo out; echo err >&2; exit 3" });
+ expect(post.body).toMatchObject({
+ stdout: "out\n",
+ stderr: "err\n",
+ exitCode: 3,
+ truncated: false,
+ });
+ expect(post.headers["x-exit-code"]).toBe("3");
+ });
+
+ it("mounts the docs and reads pages on demand", async () => {
+ const res = await run("ls /docs /docs/sdks; cat /docs/sdks/cli.md");
+ expect(res.text).toBe(
+ "/docs:\nintroduction.md\nllms.txt\nsdks\n\n/docs/sdks:\ncli.md\npage https://docs.firecrawl.dev/sdks/cli.md\n",
+ );
+ });
+
+ it("scrapes through the API's own route as the caller", async () => {
+ const res = await run("firecrawl scrape example.com --full-page");
+ expect(res.text).toBe("# Example\n");
+ expect(calls).toHaveLength(1);
+ expect(calls[0]).toMatchObject({
+ path: "/v2/scrape",
+ // Auth and quotas key on this: it must be the caller, not loopback.
+ ip: "203.0.113.9",
+ authorization: undefined,
+ body: {
+ url: "https://example.com",
+ formats: ["markdown"],
+ onlyMainContent: false,
+ origin: "bash",
+ integration: "_firecrawl-bash",
+ },
+ });
+ });
+
+ it("gives each caller its own identity", async () => {
+ await run("firecrawl scrape https://example.com", "198.51.100.1");
+ await run("firecrawl scrape https://example.com", "198.51.100.2");
+ expect(calls.map(call => call.ip)).toEqual([
+ "198.51.100.1",
+ "198.51.100.2",
+ ]);
+ });
+
+ it("passes the caller's API key through", async () => {
+ await run("firecrawl scrape https://example.com").set(
+ "Authorization",
+ "Bearer fc-test",
+ );
+ expect(calls[0].authorization).toBe("Bearer fc-test");
+ });
+
+ it("prints scrape formats the way the CLI does", async () => {
+ const res = await run(
+ [
+ "firecrawl scrape https://example.com links",
+ "firecrawl scrape https://example.com -f markdown,links",
+ "firecrawl scrape https://example.com -o out/page.md && cat out/page.md",
+ ].join("\n"),
+ );
+ expect(res.text).toBe(
+ [
+ "https://a.test\nhttps://b.test",
+ '{"markdown":"# Example","links":["https://a.test","https://b.test"],"metadata":{"title":"Example"}}',
+ "# Example",
+ ].join("\n"),
+ );
+ });
+
+ it("saves several URLs to .firecrawl/ and reports the ones that fail", async () => {
+ respond = call =>
+ call.body.url.includes("bad")
+ ? { status: 500, json: { success: false, error: "boom" } }
+ : { status: 200, json: { success: true, data: { markdown: "ok" } } };
+ const res = await run(
+ "firecrawl scrape https://good.test/a/b https://bad.test; ls .firecrawl",
+ );
+ expect(res.text).toContain("good.test-a-b.md\n");
+ expect(res.text).toContain("Scraping 2 URLs...");
+ expect(res.text).toContain("Saved: .firecrawl/good.test-a-b.md");
+ expect(res.text).toContain("Error: https://bad.test - boom");
+ expect(res.text).toContain("Completed: 1/2 succeeded, 1 failed");
+ expect(res.headers["x-exit-code"]).toBe("0");
+ });
+
+ it("searches, in the CLI's readable and JSON output", async () => {
+ const readable = await run('firecrawl search "two words" --limit 2');
+ expect(readable.text).toBe(
+ [
+ "First",
+ " URL: https://first.test",
+ " About first",
+ "",
+ "Untitled",
+ " URL: https://second.test",
+ "",
+ " --- Content ---",
+ " line 1",
+ " line 2",
+ " --- End Content ---",
+ "",
+ ].join("\n"),
+ );
+ expect(calls[0].body).toMatchObject({
+ query: "two words",
+ limit: 2,
+ sources: [{ type: "web" }],
+ });
+
+ const json = await run(
+ "firecrawl search x --json | jq -r '.id, .creditsUsed, .data.web[0].url'",
+ );
+ expect(json.text).toBe("search-1\n2\nhttps://first.test\n");
+ });
+
+ it("treats words and anything after -- as the query, not as options", async () => {
+ await run("firecrawl search help desk; firecrawl search -- --limit -h");
+ expect(calls.map(call => call.body.query)).toEqual([
+ "help desk",
+ "--limit -h",
+ ]);
+ });
+
+ it("rejects a JSON body without a command", async () => {
+ for (const body of ['{"cmd": "echo hi"}', "{not json"]) {
+ const res = await request(base).post("/v2/bash").type("json").send(body);
+ expect(res.status).toBe(400);
+ expect(res.text).toContain('{"command": "..."}');
+ }
+ });
+
+ it("shows the API's own error, such as a keyless limit, and exits 1", async () => {
+ respond = () => ({
+ status: 429,
+ json: { success: false, error: "You've hit the keyless limit." },
+ });
+ const res = await run("firecrawl search anything");
+ expect(res.text).toBe("Error: You've hit the keyless limit.\n[exit 1]\n");
+ expect(res.headers["x-exit-code"]).toBe("1");
+ });
+
+ it("explains a bare keyless refusal", async () => {
+ respond = () => ({
+ status: 401,
+ json: { success: false, error: "Unauthorized" },
+ });
+ const res = await run("firecrawl scrape https://example.com");
+ expect(res.text).toContain("Create a free API key at");
+ expect(res.headers["x-exit-code"]).toBe("1");
+ });
+
+ it("exits 2 without calling the API for anything outside the subset", async () => {
+ const res = await run(
+ [
+ "firecrawl crawl https://example.com",
+ "firecrawl scrape https://example.com --screenshot",
+ "firecrawl scrape https://example.com -f screenshot",
+ "firecrawl search x --sources alexandria",
+ ].join("; echo $?; "),
+ );
+ expect(res.text.match(/^2$/gm)).toHaveLength(3);
+ expect(res.text.match(/npx -y firecrawl-cli@latest init/g)).toHaveLength(4);
+ expect(res.headers["x-exit-code"]).toBe("2");
+ expect(calls).toHaveLength(0);
+ });
+
+ it("stops runaway commands and leaves out everything but text tools", async () => {
+ const loop = await run("while true; do :; done");
+ expect(loop.headers["x-exit-code"]).toBe("126");
+ expect(Number(loop.headers["x-duration-ms"])).toBeLessThan(5000);
+
+ const memory = await run(
+ 's=xxxxxxxx; for i in $(seq 1 22); do s="$s$s"; done; echo ${#s}',
+ );
+ expect(memory.headers["x-exit-code"]).toBe("126");
+
+ const tools = await run(
+ "sqlite3 :memory: 'select 1'; curl https://example.com; python3 -c 1",
+ );
+ expect(tools.headers["x-exit-code"]).toBe("127");
+ });
+
+ it("caps Firecrawl calls per command", async () => {
+ const res = await run(
+ "for i in $(seq 1 30); do firecrawl scrape https://example.com > /dev/null || break; done",
+ );
+ expect(calls).toHaveLength(25);
+ expect(res.text).toContain("call limit reached (25 Firecrawl calls");
+ });
+
+ it("truncates large output", async () => {
+ const res = await run(
+ 's=xxxxxxxx; for i in $(seq 1 17); do s="$s$s"; done; echo "$s"',
+ );
+ expect(res.text).toContain("[truncated at 256000 bytes]");
+ expect(res.text.length).toBeLessThan(260_000);
+ });
+
+ it("rejects an oversized command", async () => {
+ const res = await run(`echo ${"x".repeat(70_000)}`);
+ expect(res.status).toBe(413);
+ });
+
+ it("sheds a caller that sends too many commands", async () => {
+ const statuses: number[] = [];
+ for (let i = 0; i < 62; i++) {
+ const res = await request(base)
+ .get("/v2/bash")
+ .set("X-Forwarded-For", "192.0.2.77");
+ statuses.push(res.status);
+ }
+ expect(statuses.slice(0, 60).every(status => status === 200)).toBe(true);
+ expect(statuses.slice(60)).toEqual([429, 429]);
+ });
+});
diff --git a/apps/api/src/controllers/v2/bash.ts b/apps/api/src/controllers/v2/bash.ts
new file mode 100644
index 000000000..f25d0318f
--- /dev/null
+++ b/apps/api/src/controllers/v2/bash.ts
@@ -0,0 +1,311 @@
+import cors from "cors";
+import express, {
+ type Application,
+ type Request,
+ type RequestHandler,
+ type Response,
+} from "express";
+import { isIPv4 } from "net";
+import {
+ RateLimiterMemory,
+ RateLimiterRedis,
+ RateLimiterRes,
+} from "rate-limiter-flexible";
+import { config } from "../../config";
+import {
+ MAX_FIRECRAWL_CALLS,
+ MAX_OUTPUT_BYTES,
+ TIMEOUT_MS,
+} from "../../lib/hosted-bash/limits";
+import {
+ ShellBusyError,
+ runShell,
+ type ShellResult,
+} from "../../lib/hosted-bash/shell";
+import { keylessTeamId, keylessTeamUuid } from "../../lib/keyless-team";
+import { logger } from "../../lib/logger";
+import { capturePostHog } from "../../services/posthog-capture";
+import { redisRateLimitClient } from "../../services/rate-limiter";
+
+const ROUTE = "/v2/bash";
+const RUN_EVENT = "bash_command_run";
+const USAGE_EVENT = "bash_usage_viewed";
+const MAX_COMMAND_BYTES = 64 * 1024;
+const COMMANDS_PER_MINUTE = 60;
+// CLI commands worth counting by name; any other word after `firecrawl` is "other".
+const CLI_COMMANDS = new Set([
+ "search",
+ "scrape",
+ "crawl",
+ "map",
+ "extract",
+ "agent",
+ "parse",
+ "download",
+ "batch-scrape",
+ "interact",
+ "browser",
+ "monitor",
+ "login",
+ "init",
+ "help",
+]);
+
+function usage(base: string): string {
+ return `Firecrawl Bash
+
+A hosted shell for agents. Send one bash command over HTTP and get its output back.
+A subset of the Firecrawl CLI (search and scrape) is built in as \`firecrawl\`, and the
+live Firecrawl docs are mounted at /docs. No API key, MCP server or install needed.
+
+Run a command:
+ curl ${base} --data-binary 'firecrawl search "stripe pricing" --limit 3'
+ curl ${base} --data-binary 'firecrawl scrape https://stripe.com/pricing | head -40'
+ curl ${base} --data-binary 'firecrawl --help'
+ GET ${base}?command=firecrawl%20search%20stripe%20pricing
+ POST ${base} JSON: {"command": "..."}
+
+Read the Firecrawl docs:
+ curl ${base} --data-binary 'cat /docs/llms.txt | head -40'
+ curl ${base} --data-binary 'grep -l -r keyless /docs/features'
+
+Pipes, grep, jq, sed, awk and loops work, for example:
+ curl ${base} --data-binary 'firecrawl search "firecrawl changelog" --limit 5 --json | jq -r ".data.web[].url"'
+
+Response:
+ Plain text (stdout, then stderr) by default. Add ?format=json or Accept: application/json
+ for stdout, stderr, exitCode, durationMs and truncated. X-Exit-Code is set on every run.
+
+Without an API key, firecrawl runs on the free keyless tier, limited per IP per day.
+A limit message includes a link to a free API key; send one as Authorization: Bearer fc-...
+for higher limits. For crawl, map and everything else, install the Firecrawl CLI:
+npx -y firecrawl-cli@latest init
+
+Limits: ${MAX_COMMAND_BYTES} byte commands, ${MAX_OUTPUT_BYTES} bytes of stdout and stderr each,
+${TIMEOUT_MS / 1000} seconds and ${MAX_FIRECRAWL_CALLS} Firecrawl calls per command. Nothing persists between requests.
+`;
+}
+
+/**
+ * Commands per IP per minute across every replica. Firecrawl calls also count
+ * against the keyless tier or the caller's plan; commands that make none have
+ * only this limit.
+ */
+function createRateLimit(): RequestHandler {
+ const memory = new RateLimiterMemory({
+ points: COMMANDS_PER_MINUTE,
+ duration: 60,
+ });
+ const limiter = config.REDIS_RATE_LIMIT_URL
+ ? new RateLimiterRedis({
+ storeClient: redisRateLimitClient,
+ keyPrefix: "hosted-bash",
+ points: COMMANDS_PER_MINUTE,
+ duration: 60,
+ insuranceLimiter: memory,
+ })
+ : memory;
+ return async (req, res, next) => {
+ try {
+ await limiter.consume(req.ip ?? "unknown");
+ next();
+ } catch (rejection) {
+ if (!(rejection instanceof RateLimiterRes)) return next(rejection);
+ const seconds = Math.max(1, Math.ceil(rejection.msBeforeNext / 1000));
+ res.setHeader("Retry-After", String(seconds));
+ res
+ .status(429)
+ .type("text/plain")
+ .send(`Too many commands. Try again in ${seconds} seconds.\n`);
+ }
+ };
+}
+
+/** The command sent, undefined when none was, or null for a JSON body without one. */
+function readCommand(req: Request): string | null | undefined {
+ const fromQuery =
+ typeof req.query.command === "string" ? req.query.command : undefined;
+ const body = typeof req.body === "string" ? req.body : "";
+ if (!body.trim()) return fromQuery;
+ const json = req.is("application/json");
+ // A JSON object is a request even without its content type; a brace group
+ // like `{ echo a; }` does not parse, so it stays a command.
+ if (json || body.trimStart().startsWith("{")) {
+ try {
+ const parsed = JSON.parse(body);
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+ return typeof parsed.command === "string" ? parsed.command : null;
+ }
+ } catch {}
+ if (json) return null;
+ }
+ // curl --data-urlencode 'command=...' sends a form; --data-binary also labels
+ // a raw command as a form, so only the command= field is decoded.
+ if (
+ req.is("application/x-www-form-urlencoded") &&
+ body.startsWith("command=")
+ ) {
+ return new URLSearchParams(body).get("command") ?? "";
+ }
+ return body;
+}
+
+/** Where this server answered the request, so the shell can call its routes. */
+function ownBaseUrl(req: Request): string {
+ const { localAddress, localPort } = req.socket;
+ const host = !localAddress
+ ? "127.0.0.1"
+ : localAddress.startsWith("::ffff:")
+ ? localAddress.slice(7)
+ : localAddress.includes(":")
+ ? `[${localAddress}]`
+ : localAddress;
+ return `http://${host}:${localPort}`;
+}
+
+/** Records a usage view, or the shape of a run (subcommands, outcome), never the command text. */
+function track(
+ req: Request,
+ ownKey: boolean,
+ run?: { command: string; result: ShellResult },
+) {
+ const ip = (req.ip ?? "").replace(/^::ffff:/i, "");
+ // The id keyless_prompt_shown uses, so keyless runs join to prompts and signups.
+ const keylessTeam =
+ !ownKey && isIPv4(ip) ? keylessTeamUuid(keylessTeamId(ip)) : null;
+ const event = run ? RUN_EVENT : USAGE_EVENT;
+ const properties = {
+ keyless_team_id: keylessTeam,
+ user_agent: req.get("user-agent")?.slice(0, 300) ?? null,
+ method: req.method,
+ own_key: ownKey,
+ ...(run
+ ? {
+ subcommands: [
+ ...new Set(
+ [...run.command.matchAll(/\bfirecrawl\s+([a-z][\w-]*)/g)].map(
+ match => (CLI_COMMANDS.has(match[1]) ? match[1] : "other"),
+ ),
+ ),
+ ],
+ exit_code: run.result.exitCode,
+ duration_ms: run.result.durationMs,
+ endpoints: run.result.endpoints,
+ failed_endpoints: run.result.failedEndpoints,
+ truncated: run.result.truncated,
+ }
+ : {}),
+ };
+ logger.info("Hosted bash request", {
+ module: "hosted-bash",
+ event,
+ ...properties,
+ });
+ void capturePostHog(event, keylessTeam ?? "hosted-bash-anonymous", {
+ ...properties,
+ // Callers are anonymous IPs: do not create a person for each one.
+ $process_person_profile: false,
+ });
+}
+
+async function bashController(req: Request, res: Response) {
+ res.setHeader("Cache-Control", "no-store");
+ const sent = req.method === "HEAD" ? undefined : readCommand(req);
+ const authorization = req.get("authorization")?.trim() || null;
+
+ if (sent === null) {
+ res
+ .status(400)
+ .type("text/plain")
+ .send('Send JSON as {"command": "..."}, or the command as the body.\n');
+ return;
+ }
+ const command = sent?.trim();
+ if (!command) {
+ track(req, Boolean(authorization));
+ res
+ .type("text/plain")
+ .send(usage(`${req.protocol}://${req.get("host")}${ROUTE}`));
+ return;
+ }
+ if (Buffer.byteLength(command) > MAX_COMMAND_BYTES) {
+ res
+ .status(413)
+ .type("text/plain")
+ .send(`command is larger than ${MAX_COMMAND_BYTES} bytes\n`);
+ return;
+ }
+
+ const userAgent = req.get("user-agent")?.slice(0, 500);
+ const worldId = req.get("x-firecrawl-world-id");
+ const closed = new AbortController();
+ res.on("close", () => {
+ if (!res.writableFinished) closed.abort();
+ });
+ let result: ShellResult;
+ try {
+ result = await runShell(
+ command,
+ {
+ baseUrl: ownBaseUrl(req),
+ ip: req.ip ?? null,
+ authorization,
+ headers: {
+ ...(userAgent ? { "User-Agent": userAgent } : {}),
+ ...(worldId ? { "X-Firecrawl-World-Id": worldId } : {}),
+ },
+ },
+ closed.signal,
+ );
+ } catch (error) {
+ if (!(error instanceof ShellBusyError)) throw error;
+ res
+ .status(503)
+ .setHeader("Retry-After", "5")
+ .type("text/plain")
+ .send(
+ "Every shell on this server is busy. Try again in a few seconds.\n",
+ );
+ return;
+ }
+ track(req, Boolean(authorization), { command, result });
+
+ res.setHeader("X-Exit-Code", String(result.exitCode));
+ res.setHeader("X-Duration-Ms", String(result.durationMs));
+ if (req.query.format === "json" || req.accepts(["text", "json"]) === "json") {
+ const { stdout, stderr, exitCode, durationMs, truncated } = result;
+ res.json({ stdout, stderr, exitCode, durationMs, truncated });
+ return;
+ }
+ let body = result.stdout;
+ if (result.stderr) {
+ body += (body && !body.endsWith("\n") ? "\n" : "") + result.stderr;
+ }
+ if (result.exitCode !== 0) {
+ body += `${body && !body.endsWith("\n") ? "\n" : ""}[exit ${result.exitCode}]\n`;
+ }
+ res.type("text/plain").send(body);
+}
+
+/**
+ * Hosted bash for agents: one command per request, in a fresh sandboxed shell.
+ * Registered ahead of the global body parsers so the body is read once, as
+ * text, under this route's own size limit.
+ */
+export function registerHostedBashRoute(
+ app: Pick<Application, "get" | "post" | "options">,
+) {
+ const allowCors = cors({
+ exposedHeaders: ["X-Exit-Code", "X-Duration-Ms", "Retry-After"],
+ });
+ const rateLimit = createRateLimit();
+ app.options(ROUTE, allowCors);
+ app.get(ROUTE, allowCors, rateLimit, bashController);
+ app.post(
+ ROUTE,
+ allowCors,
+ rateLimit,
+ express.text({ type: () => true, limit: MAX_COMMAND_BYTES }),
+ bashController,
+ );
+}
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 87dfa9a31..da6ddcf79 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2";
import { exchangeRouter } from "./routes/exchange";
import { labsRouter } from "./routes/labs";
import { registerMcpActionLogIngestRoute } from "./routes/mcp-action-logs";
+import { registerHostedBashRoute } from "./controllers/v2/bash";
import { nuqShutdown } from "./services/worker/nuq";
import { getErrorContactMessage } from "./lib/deployment";
import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
@@ -91,6 +92,7 @@ const captureRawBody = (
};
registerMcpActionLogIngestRoute(app);
+registerHostedBashRoute(app);
app.use(bodyParser.urlencoded({ extended: true, verify: captureRawBody }));
app.use(bodyParser.json({ limit: "10mb", verify: captureRawBody }));
diff --git a/apps/api/src/lib/hosted-bash/docs.test.ts b/apps/api/src/lib/hosted-bash/docs.test.ts
new file mode 100644
index 000000000..8edad7b40
--- /dev/null
+++ b/apps/api/src/lib/hosted-bash/docs.test.ts
@@ -0,0 +1,85 @@
+const ROOT = "https://docs.firecrawl.dev/llms.txt";
+const SUB = "https://docs.firecrawl.dev/_llms/en/v2.txt";
+const ROOT_INDEX = `- [Sub](${SUB}): more.
+- [Intro](https://docs.firecrawl.dev/introduction.md): Start here.
+`;
+const SUB_INDEX = `- [CLI](https://docs.firecrawl.dev/sdks/cli.md): The CLI.
+`;
+
+describe("hosted bash docs mount", () => {
+ let failing: Set<string>;
+ let fetched: string[];
+
+ // A fresh module per test: the page list is cached at module level.
+ async function load() {
+ vi.resetModules();
+ return (await import("./docs.js")).docsFiles;
+ }
+
+ beforeEach(() => {
+ failing = new Set();
+ fetched = [];
+ vi.spyOn(globalThis, "fetch").mockImplementation(input => {
+ const url = String(input);
+ fetched.push(url);
+ if (failing.has(url)) return Promise.reject(new Error("docs down"));
+ return Promise.resolve(
+ new Response(url === ROOT ? ROOT_INDEX : SUB_INDEX),
+ );
+ });
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ vi.restoreAllMocks();
+ });
+
+ it("lists pages from the root index and its English sub-indexes", async () => {
+ const docsFiles = await load();
+ expect(Object.keys(await docsFiles()).sort()).toEqual([
+ "/docs/introduction.md",
+ "/docs/llms.txt",
+ "/docs/sdks/cli.md",
+ ]);
+ });
+
+ it("keeps the pages it found when one sub-index fails", async () => {
+ failing.add(SUB);
+ const docsFiles = await load();
+ expect(Object.keys(await docsFiles()).sort()).toEqual([
+ "/docs/introduction.md",
+ "/docs/llms.txt",
+ ]);
+ });
+
+ it("starts with an empty /docs when the docs site is down, and retries later", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ failing.add(ROOT);
+ const docsFiles = await load();
+ expect(Object.keys(await docsFiles())).toEqual(["/docs/llms.txt"]);
+
+ // Within the retry window a command does not touch the docs site at all.
+ fetched.length = 0;
+ await docsFiles();
+ expect(fetched).toEqual([]);
+
+ failing.clear();
+ vi.setSystemTime(Date.now() + 61_000);
+ await docsFiles();
+ await vi.waitFor(async () =>
+ expect(Object.keys(await docsFiles())).toHaveLength(3),
+ );
+ });
+
+ it("serves the last list while a later refresh fails", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ const docsFiles = await load();
+ expect(Object.keys(await docsFiles())).toHaveLength(3);
+
+ failing.add(ROOT).add(SUB);
+ vi.setSystemTime(Date.now() + 2 * 60 * 60 * 1000);
+ expect(Object.keys(await docsFiles())).toHaveLength(3);
+ await new Promise(resolve => setImmediate(resolve));
+ expect(Object.keys(await docsFiles())).toHaveLength(3);
+ });
+});
diff --git a/apps/api/src/lib/hosted-bash/docs.ts b/apps/api/src/lib/hosted-bash/docs.ts
new file mode 100644
index 000000000..79378a9bc
--- /dev/null
+++ b/apps/api/src/lib/hosted-bash/docs.ts
@@ -0,0 +1,108 @@
+import type { InitialFiles } from "just-bash";
+import { logger } from "../logger";
+
+const DOCS_ORIGIN = "https://docs.firecrawl.dev";
+const INDEX_URL = `${DOCS_ORIGIN}/llms.txt`;
+const TTL_MS = 60 * 60 * 1000;
+const RETRY_MS = 60 * 1000;
+const FETCH_TIMEOUT_MS = 10_000;
+const LINK = /\]\((https:\/\/docs\.firecrawl\.dev\/[^)\s]+)\)/g;
+
+/** Docs text by URL, shared by every shell on this server for an hour. */
+const cache = new Map<string, { at: number; text: Promise<string> }>();
+
+function fetchText(url: string): Promise<string> {
+ const hit = cache.get(url);
+ if (hit && Date.now() - hit.at < TTL_MS) return hit.text;
+ const text = fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })
+ .then(res => {
+ if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);
+ return res.text();
+ })
+ .catch(error => {
+ // A failure is not cached, so the next read tries again.
+ if (cache.get(url)?.text === text) cache.delete(url);
+ throw error;
+ });
+ cache.set(url, { at: Date.now(), text });
+ return text;
+}
+
+/**
+ * Every English page path, found by walking llms.txt and its English
+ * sub-indexes. An index that fails to load is skipped and reported, so the
+ * pages found elsewhere are kept.
+ */
+async function walkIndexes(): Promise<{ pages: string[]; complete: boolean }> {
+ const seen = new Set<string>();
+ const pages = new Set<string>();
+ let complete = true;
+ const walk = async (url: string): Promise<void> => {
+ if (seen.has(url)) return;
+ seen.add(url);
+ let index: string;
+ try {
+ index = await fetchText(url);
+ } catch (error) {
+ complete = false;
+ logger.warn("Hosted bash could not load a docs index", {
+ module: "hosted-bash",
+ url,
+ error,
+ });
+ return;
+ }
+ const links = [...index.matchAll(LINK)].map(m => m[1]);
+ for (const link of links) {
+ if (link.endsWith(".md")) pages.add(new URL(link).pathname);
+ }
+ await Promise.all(
+ links
+ .filter(
+ link =>
+ link.endsWith(".txt") &&
+ new URL(link).pathname.startsWith("/_llms/en"),
+ )
+ .map(walk),
+ );
+ };
+ await walk(INDEX_URL);
+ return { pages: [...pages].sort(), complete };
+}
+
+let known: string[] | null = null;
+let refreshAt = 0;
+let refresh: Promise<string[]> | null = null;
+
+/**
+ * The page list, refreshed hourly in the background. A command only waits for
+ * it on a server that has never listed the docs; after that a slow or failing
+ * docs site leaves the last list in place and is retried a minute later.
+ */
+function listPages(): Promise<string[]> {
+ if (!refresh && Date.now() >= refreshAt) {
+ refresh = walkIndexes()
+ .then(({ pages, complete }) => {
+ refreshAt = Date.now() + (complete ? TTL_MS : RETRY_MS);
+ // A partial walk never replaces a fuller list.
+ if (complete || pages.length >= (known?.length ?? 0)) known = pages;
+ return known ?? pages;
+ })
+ .finally(() => {
+ refresh = null;
+ });
+ }
+ return known ? Promise.resolve(known) : (refresh as Promise<string[]>);
+}
+
+/**
+ * The live docs.firecrawl.dev pages as files under /docs. Only the page list is
+ * needed to start a shell; a page is fetched when a command first reads it.
+ */
+export async function docsFiles(): Promise<InitialFiles> {
+ const files: InitialFiles = { "/docs/llms.txt": () => fetchText(INDEX_URL) };
+ for (const path of await listPages().catch(() => [])) {
+ files[`/docs${path}`] = () => fetchText(`${DOCS_ORIGIN}${path}`);
+ }
+ return files;
+}
diff --git a/apps/api/src/lib/hosted-bash/firecrawl-command.ts b/apps/api/src/lib/hosted-bash/firecrawl-command.ts
new file mode 100644
index 000000000..15155ac83
--- /dev/null
+++ b/apps/api/src/lib/hosted-bash/firecrawl-command.ts
@@ -0,0 +1,657 @@
+import { defineCommand, type IFileSystem } from "just-bash";
+
+const ORIGIN = "bash";
+const INTEGRATION = "_firecrawl-bash";
+const INSTALL = "npx -y firecrawl-cli@latest init";
+const SEARCH_SOURCES = ["web", "images", "news"];
+const SEARCH_CATEGORIES = ["research", "pdf", "developer", "gov"];
+// Every format the CLI accepts; the hosted shell serves SCRAPE_FORMATS of them.
+const CLI_FORMATS = [
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "images",
+ "screenshot",
+ "summary",
+ "changeTracking",
+ "json",
+ "attributes",
+ "branding",
+];
+const SCRAPE_FORMATS = [
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "images",
+ "summary",
+];
+const API_KEY_FLAG =
+ "--api-key isn't accepted in the hosted shell. Send your key in the HTTP request's Authorization: Bearer fc-... header instead";
+const KEY_NEEDED =
+ "Firecrawl's free tier without an API key isn't available from your network (it needs an IPv4 address). Create a free API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer fc-...";
+
+export interface FirecrawlCaller {
+ /** This server's own address: calls go through the same routes a REST caller uses. */
+ baseUrl: string;
+ ip: string | null;
+ /** The caller's Authorization header as sent, so the API parses it as it would directly. */
+ authorization: string | null;
+ /** Caller headers kept on each call: User-Agent and a World ID credential. */
+ headers: Record<string, string>;
+ maxCalls: number;
+ /** When the command must be done; each call asks the API to finish before it. */
+ deadline: number;
+}
+
+// Room to read the response and print it before the command's deadline.
+const CALL_MARGIN_MS = 3_000;
+
+type Endpoint = "search" | "scrape";
+
+/** Calls the API as the shell's caller and records what was called, for analytics. */
+export class FirecrawlClient {
+ readonly endpoints: Endpoint[] = [];
+ readonly failedEndpoints: Endpoint[] = [];
+
+ constructor(private readonly caller: FirecrawlCaller) {}
+
+ async post(
+ endpoint: Endpoint,
+ body: Record<string, unknown>,
+ signal?: AbortSignal,
+ ): Promise<any> {
+ const { baseUrl, ip, authorization, headers, maxCalls, deadline } =
+ this.caller;
+ if (this.endpoints.length >= maxCalls) {
+ throw new Error(
+ `call limit reached (${maxCalls} Firecrawl calls per command)`,
+ );
+ }
+ // A call the API is still working on at the deadline would be billed but lost.
+ const timeout = deadline - Date.now() - CALL_MARGIN_MS;
+ if (timeout < 1_000) {
+ throw new Error("not enough time left before the command's deadline");
+ }
+ this.endpoints.push(endpoint);
+ const res = await fetch(`${baseUrl}/v2/${endpoint}`, {
+ method: "POST",
+ headers: {
+ ...headers,
+ "Content-Type": "application/json",
+ ...(authorization ? { Authorization: authorization } : {}),
+ // This request arrives from loopback, a trusted hop under
+ // EXPRESS_TRUST_PROXY, so auth and quotas see the caller's IP.
+ ...(ip ? { "X-Forwarded-For": ip } : {}),
+ },
+ body: JSON.stringify({
+ ...body,
+ timeout,
+ origin: ORIGIN,
+ integration: INTEGRATION,
+ }),
+ signal,
+ });
+ const json: any = await res.json().catch(() => ({}));
+ if (!res.ok || json?.success === false) {
+ this.failedEndpoints.push(endpoint);
+ // Keyless refuses an IPv6 caller with a bare 401.
+ const bare = !json?.error || json.error === "Unauthorized";
+ if (!authorization && res.status === 401 && bare) {
+ throw new Error(KEY_NEEDED);
+ }
+ throw new Error(apiError(json, res.status));
+ }
+ return json;
+ }
+}
+
+/** The API's error, with the first few validation issues it lists. */
+function apiError(json: any, status: number): string {
+ const message = json?.error || `Firecrawl request failed (HTTP ${status})`;
+ const issues: string[] = Array.isArray(json?.details)
+ ? json.details
+ .slice(0, 3)
+ .filter((d: any) => typeof d?.message === "string")
+ .map((d: any) =>
+ Array.isArray(d.path) && d.path.length
+ ? `${d.path.join(".")}: ${d.message}`
+ : d.message,
+ )
+ : [];
+ return issues.length ? `${message} (${issues.join("; ")})` : message;
+}
+
+export const FIRECRAWL_HELP = `Usage: firecrawl [command] [options]
+
+A hosted subset of the Firecrawl CLI. The flags and output follow the real CLI
+(/docs/sdks/cli.md); install it for everything else: ${INSTALL}
+
+Commands:
+ search <query> Search the web
+ --limit <number> Maximum number of results (default: 5)
+ --sources <sources> Comma-separated sources: web, images, news (default: web)
+ --categories <categories> Comma-separated categories: ${SEARCH_CATEGORIES.join(", ")}
+ --tbs <value> Time-based search: qdr:h, qdr:d, qdr:w, qdr:m, qdr:y
+ --location <location> Location for geo-targeting
+ --country <code> ISO country code for geo-targeting
+ --scrape Enable scraping of search results
+ --scrape-formats <formats> Comma-separated scrape formats when --scrape is enabled (default: markdown)
+ -o, --output <path> Output file path (default: stdout)
+ --json Output as compact JSON
+ --pretty Output as pretty JSON
+
+ scrape [urls...] Scrape URLs. Multiple URLs are saved to .firecrawl/
+ -f, --format <formats> Output format(s): ${SCRAPE_FORMATS.join(", ")}.
+ Single format outputs raw content; multiple formats output JSON.
+ -H, --html Output raw HTML (shortcut for --format html)
+ -S, --summary Output summary (shortcut for --format summary)
+ --only-main-content Include only the main content (the API default)
+ --full-page Keep the whole page, including navigation, headers and footers
+ --wait-for <ms> Wait time before scraping in milliseconds
+ --include-tags <tags> Comma-separated list of tags to include
+ --exclude-tags <tags> Comma-separated list of tags to exclude
+ --max-age <milliseconds> Maximum age of cached content in milliseconds
+ --country <code> ISO country code for geo-targeted scraping
+ -o, --output <path> Output file path (default: stdout)
+ --json Output as JSON format
+ --pretty Pretty print JSON output
+
+ help Display this help
+`;
+
+/** Thrown for anything the hosted subset leaves out; exits 2, unlike a failed call. */
+class Unsupported extends Error {}
+
+type Flags = Record<string, string | true>;
+
+interface FlagSpec {
+ values: string[];
+ booleans: string[];
+ aliases: Record<string, string>;
+}
+
+const SEARCH_FLAGS: FlagSpec = {
+ values: [
+ "limit",
+ "sources",
+ "categories",
+ "tbs",
+ "location",
+ "country",
+ "scrape-formats",
+ "output",
+ ],
+ booleans: ["scrape", "json", "pretty"],
+ aliases: { o: "output" },
+};
+
+const SCRAPE_FLAGS: FlagSpec = {
+ values: [
+ "url",
+ "format",
+ "wait-for",
+ "include-tags",
+ "exclude-tags",
+ "max-age",
+ "country",
+ "output",
+ ],
+ booleans: [
+ "html",
+ "summary",
+ "only-main-content",
+ "full-page",
+ "json",
+ "pretty",
+ ],
+ aliases: {
+ u: "url",
+ f: "format",
+ formats: "format",
+ H: "html",
+ S: "summary",
+ o: "output",
+ },
+};
+
+function parseArgs(args: string[], spec: FlagSpec) {
+ const positional: string[] = [];
+ const flags: Flags = {};
+ for (let i = 0; i < args.length; i++) {
+ const arg = args[i];
+ if (arg === "--") {
+ positional.push(...args.slice(i + 1));
+ break;
+ }
+ const match = /^(?:--([\w-]+)|-([A-Za-z]))(?:=(.*))?$/s.exec(arg);
+ if (!match) {
+ positional.push(arg);
+ continue;
+ }
+ const raw = match[1] ?? match[2];
+ const name = spec.aliases[raw] ?? raw;
+ const shown = match[1] ? `--${raw}` : `-${raw}`;
+ if (spec.booleans.includes(name)) {
+ flags[name] = true;
+ } else if (!spec.values.includes(name)) {
+ throw new Unsupported(`option '${shown}' isn't in the hosted shell`);
+ } else if (match[3] !== undefined) {
+ flags[name] = match[3];
+ } else if (i + 1 < args.length) {
+ flags[name] = args[++i];
+ } else {
+ throw new Error(`option '${shown}' argument missing`);
+ }
+ }
+ return { positional, flags };
+}
+
+const text = (flags: Flags, name: string) =>
+ typeof flags[name] === "string" ? (flags[name] as string) : undefined;
+
+function integer(flags: Flags, name: string): number | undefined {
+ const value = text(flags, name);
+ if (value === undefined) return undefined;
+ const parsed = Number.parseInt(value, 10);
+ if (Number.isNaN(parsed)) {
+ throw new Error(`option '--${name}' expects a number`);
+ }
+ return parsed;
+}
+
+const list = (value: string | undefined) =>
+ value
+ ?.split(",")
+ .map(item => item.trim())
+ .filter(Boolean);
+
+interface Context {
+ client: FirecrawlClient;
+ fs: IFileSystem;
+ cwd: string;
+ signal?: AbortSignal;
+ stderr: string[];
+}
+
+/** The CLI's writeOutput: a file when a path is given, otherwise stdout ending in a newline. */
+async function output(c: Context, content: string, path: string | undefined) {
+ if (!path) return content.endsWith("\n") ? content : `${content}\n`;
+ const absolute = path.startsWith("/")
+ ? path
+ : `${c.cwd.replace(/\/$/, "")}/${path}`;
+ const dir = absolute.slice(0, absolute.lastIndexOf("/")) || "/";
+ await c.fs.mkdir(dir, { recursive: true });
+ await c.fs.writeFile(absolute, content);
+ return "";
+}
+
+const wantsJson = (flags: Flags, path: string | undefined) =>
+ Boolean(flags.json) || Boolean(path?.toLowerCase().endsWith(".json"));
+
+const stringify = (flags: Flags, value: unknown) =>
+ flags.pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value);
+
+function scrapedContent(markdown: unknown): string[] {
+ if (typeof markdown !== "string" || !markdown) return [];
+ const indented = markdown
+ .split("\n")
+ .map(line => ` ${line}`)
+ .join("\n");
+ return ["", " --- Content ---", indented, " --- End Content ---"];
+}
+
+function readableSearch(data: any, labelWeb: boolean): string {
+ const lines: string[] = [];
+ if (data.web?.length) {
+ if (labelWeb) lines.push("=== Web Results ===", "");
+ for (const r of data.web) {
+ lines.push(r.title || "Untitled", ` URL: ${r.url}`);
+ if (r.description) lines.push(` ${r.description}`);
+ if (r.category) lines.push(` Category: ${r.category}`);
+ lines.push(...scrapedContent(r.markdown), "");
+ }
+ }
+ if (data.images?.length) {
+ if (lines.length) lines.push("");
+ lines.push("=== Image Results ===", "");
+ for (const r of data.images) {
+ lines.push(
+ r.title || "Untitled",
+ ` Image URL: ${r.imageUrl}`,
+ ` Source: ${r.url}`,
+ );
+ if (r.imageWidth && r.imageHeight) {
+ lines.push(` Size: ${r.imageWidth}x${r.imageHeight}`);
+ }
+ lines.push("");
+ }
+ }
+ if (data.news?.length) {
+ if (lines.length) lines.push("");
+ lines.push("=== News Results ===", "");
+ for (const r of data.news) {
+ lines.push(r.title || "Untitled", ` URL: ${r.url}`);
+ if (r.date) lines.push(` Date: ${r.date}`);
+ if (r.snippet) lines.push(` ${r.snippet}`);
+ lines.push(...scrapedContent(r.markdown), "");
+ }
+ }
+ return lines.join("\n");
+}
+
+async function search(c: Context, args: string[]): Promise<string> {
+ const { positional, flags } = parseArgs(args, SEARCH_FLAGS);
+ const query = positional.join(" ");
+ if (!query) throw new Error("missing required argument 'query'");
+ const named = list(text(flags, "sources"))?.map(s => s.toLowerCase());
+ const sources = named?.length ? [...new Set(named)] : ["web"];
+ for (const source of sources) {
+ if (!SEARCH_SOURCES.includes(source)) {
+ throw new Unsupported(
+ `source "${source}" isn't in the hosted shell, which has ${SEARCH_SOURCES.join(", ")}`,
+ );
+ }
+ }
+ const categories = list(text(flags, "categories"))?.map(c => c.toLowerCase());
+ for (const category of categories ?? []) {
+ if (category === "github") {
+ throw new Error(
+ "The github category was removed. Use --categories developer",
+ );
+ }
+ if (!SEARCH_CATEGORIES.includes(category)) {
+ throw new Error(
+ `Invalid category "${category}". Valid categories are: ${SEARCH_CATEGORIES.join(", ")}`,
+ );
+ }
+ }
+ const scrapeFormats = list(text(flags, "scrape-formats")) ?? ["markdown"];
+ const optional = (key: string, flag = key) =>
+ text(flags, flag) ? { [key]: text(flags, flag) } : {};
+
+ const envelope = await c.client.post(
+ "search",
+ {
+ query,
+ limit: integer(flags, "limit") ?? 5,
+ sources: sources.map(type => ({ type })),
+ // The CLI gets highlighted descriptions by default; ask for the same.
+ highlights: true,
+ ...(categories ? { categories: categories.map(type => ({ type })) } : {}),
+ ...optional("tbs"),
+ ...optional("location"),
+ ...optional("country"),
+ ...(flags.scrape
+ ? { scrapeOptions: { formats: scrapeFormats.map(type => ({ type })) } }
+ : {}),
+ },
+ c.signal,
+ );
+ const data = envelope.data ?? {};
+ const path = text(flags, "output");
+ if (!data.web?.length && !data.images?.length && !data.news?.length) {
+ return "No results found.\n";
+ }
+ if (flags.json || flags.pretty) {
+ const result = {
+ success: true,
+ data,
+ ...(envelope.warning ? { warning: envelope.warning } : {}),
+ ...(envelope.id ? { id: envelope.id } : {}),
+ ...(envelope.creditsUsed !== undefined
+ ? { creditsUsed: envelope.creditsUsed }
+ : {}),
+ };
+ return output(c, stringify(flags, result), path);
+ }
+ return output(c, readableSearch(data, sources.length > 1), path);
+}
+
+function scrapeBody(url: string, formats: string[], flags: Flags) {
+ const waitFor = integer(flags, "wait-for");
+ const maxAge = integer(flags, "max-age");
+ const includeTags = list(text(flags, "include-tags"));
+ const excludeTags = list(text(flags, "exclude-tags"));
+ const country = text(flags, "country");
+ return {
+ url,
+ formats,
+ ...(flags["full-page"]
+ ? { onlyMainContent: false }
+ : flags["only-main-content"]
+ ? { onlyMainContent: true }
+ : {}),
+ ...(waitFor !== undefined ? { waitFor } : {}),
+ ...(includeTags ? { includeTags } : {}),
+ ...(excludeTags ? { excludeTags } : {}),
+ ...(maxAge !== undefined ? { maxAge } : {}),
+ ...(country ? { location: { country } } : {}),
+ };
+}
+
+/** A single text format prints as raw content, as the CLI's extractContent does. */
+function rawContent(data: any, format: string): string | null {
+ const value =
+ format === "html" || format === "rawHtml"
+ ? data.html || data.rawHtml
+ : data[format];
+ if (Array.isArray(value)) return value.join("\n");
+ return typeof value === "string" && value ? value : null;
+}
+
+/** The CLI's parseFormats: comma-separated, any case, in the API's casing. */
+function parseFormats(value: string): string[] {
+ const formats = value
+ .split(",")
+ .map(format => format.trim())
+ .filter(Boolean)
+ .map(format => {
+ const known = CLI_FORMATS.find(
+ name => name.toLowerCase() === format.toLowerCase(),
+ );
+ if (!known) {
+ throw new Error(
+ `Invalid format(s): ${format}. Valid formats are: ${CLI_FORMATS.join(", ")}`,
+ );
+ }
+ return known;
+ });
+ return [...new Set(formats)];
+}
+
+function isFormatList(value: string): boolean {
+ try {
+ return parseFormats(value).length > 0;
+ } catch {
+ return false;
+ }
+}
+
+const normalizeUrl = (url: string) =>
+ /^https?:\/\//i.test(url) ? url : `https://${url}`;
+
+/** The CLI's isScrapeUrl: an http(s) URL, or a host with a dot, localhost or an IPv6 literal. */
+function isScrapeUrl(value: string): boolean {
+ if (!/^https?:\/\//i.test(value)) {
+ const host = value.split(/[/?#]/, 1)[0];
+ const hostLike =
+ host.includes(".") ||
+ /^localhost(?::\d+)?$/i.test(host) ||
+ host.startsWith("[");
+ if (!hostLike) return false;
+ }
+ try {
+ return Boolean(new URL(normalizeUrl(value)).hostname);
+ } catch {
+ return false;
+ }
+}
+
+/** The CLI's isUrl, which decides when `firecrawl <url>` means scrape. */
+function isShorthandUrl(value: string): boolean {
+ if (/^https?:\/\//i.test(value)) return true;
+ return (
+ value.includes(".") &&
+ !value.startsWith("-") &&
+ /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/.test(
+ value,
+ )
+ );
+}
+
+function urlToFilename(url: string): string {
+ try {
+ const parsed = new URL(url);
+ const host = parsed.hostname.replace(/^www\./, "");
+ const path = parsed.pathname.replace(/^\/|\/$/g, "").replace(/\//g, "-");
+ return path ? `${host}-${path}.md` : `${host}.md`;
+ } catch {
+ return `${url.replace(/[^a-zA-Z0-9.-]/g, "_")}.md`;
+ }
+}
+
+async function scrape(c: Context, args: string[]): Promise<string> {
+ const { positional, flags } = parseArgs(args, SCRAPE_FLAGS);
+ // As in the CLI, each word is a URL or formats: `firecrawl scrape example.com markdown,links`.
+ const urls: string[] = [];
+ const positionalFormats: string[] = [];
+ for (const word of [
+ ...positional,
+ ...(text(flags, "url") ? [text(flags, "url") as string] : []),
+ ]) {
+ if (isScrapeUrl(word)) {
+ const url = normalizeUrl(word);
+ if (!urls.includes(url)) urls.push(url);
+ } else if (isFormatList(word))
+ positionalFormats.push(...parseFormats(word));
+ else {
+ throw new Error(
+ `${JSON.stringify(word)} isn't a URL. For a website, use an http(s) URL, for example https://example.com`,
+ );
+ }
+ }
+ if (!urls.length) {
+ throw new Error(
+ "Provide a URL to scrape, e.g. firecrawl scrape https://example.com",
+ );
+ }
+ if (flags["only-main-content"] && flags["full-page"]) {
+ throw new Error("Use either --only-main-content or --full-page, not both");
+ }
+ const formats = positionalFormats.length
+ ? [...new Set(positionalFormats)]
+ : flags.html
+ ? ["html"]
+ : flags.summary
+ ? ["summary"]
+ : text(flags, "format")
+ ? parseFormats(text(flags, "format") as string)
+ : ["markdown"];
+ for (const format of formats) {
+ if (!SCRAPE_FORMATS.includes(format)) {
+ throw new Unsupported(
+ `format "${format}" isn't in the hosted shell, which has ${SCRAPE_FORMATS.join(", ")}`,
+ );
+ }
+ }
+
+ if (urls.length > 1) {
+ c.stderr.push(`Scraping ${urls.length} URLs...`);
+ let done = 0;
+ let failed = 0;
+ await Promise.all(
+ urls.map(async url => {
+ try {
+ const body = scrapeBody(url, formats, flags);
+ const data = (await c.client.post("scrape", body, c.signal)).data;
+ const path = `.firecrawl/${urlToFilename(url)}`;
+ // As the CLI does: markdown, or the whole response, whatever the format.
+ await output(c, data?.markdown || JSON.stringify(data ?? {}), path);
+ c.stderr.push(`[${++done}/${urls.length}] Saved: ${path}`);
+ } catch (error) {
+ failed++;
+ c.stderr.push(
+ `[${++done}/${urls.length}] Error: ${url} - ${(error as Error).message}`,
+ );
+ }
+ }),
+ );
+ c.stderr.push(
+ "",
+ `Completed: ${urls.length - failed}/${urls.length} succeeded${failed ? `, ${failed} failed` : ""}`,
+ );
+ if (failed === urls.length) throw new Error("every URL failed");
+ return "";
+ }
+
+ const body = scrapeBody(urls[0], formats, flags);
+ const data = (await c.client.post("scrape", body, c.signal)).data ?? {};
+ const path = text(flags, "output");
+ if (wantsJson(flags, path)) return output(c, stringify(flags, data), path);
+ const raw = formats.length === 1 ? rawContent(data, formats[0]) : null;
+ if (raw !== null) return output(c, raw, path);
+ const picked = Object.fromEntries(
+ [...formats, "metadata"]
+ .filter(key => data[key] !== undefined)
+ .map(key => [key, data[key]]),
+ );
+ return output(
+ c,
+ stringify(flags, formats.length === 1 ? data : picked),
+ path,
+ );
+}
+
+export function firecrawlCommand(client: FirecrawlClient) {
+ return defineCommand("firecrawl", async (argv, ctx) => {
+ // As in the CLI, `firecrawl <url>` is short for `firecrawl scrape <url>`.
+ const [name, ...rest] = isShorthandUrl(argv[0] ?? "")
+ ? ["scrape", ...argv]
+ : argv;
+ const stderr: string[] = [];
+ const result = (stdout: string, exitCode: number) => ({
+ stdout,
+ stdoutKind: "text" as const,
+ stderr: stderr.length ? `${stderr.join("\n")}\n` : "",
+ exitCode,
+ });
+ // A bare `help` is only the help command up front: `search help desk` is a query.
+ const end = rest.indexOf("--");
+ const options = end === -1 ? rest : rest.slice(0, end);
+ if ([name, ...options].some(arg => /^(--api-key|-k)(=|$)/.test(arg))) {
+ stderr.push(`Error: ${API_KEY_FLAG}`);
+ return result("", 1);
+ }
+ const asksHelp = options.some(arg => arg === "--help" || arg === "-h");
+ if (!name || ["help", "--help", "-h"].includes(name) || asksHelp) {
+ return result(FIRECRAWL_HELP, 0);
+ }
+ const hint = `Install the Firecrawl CLI for the rest: ${INSTALL} (see /docs/sdks/cli.md)`;
+ if (name !== "search" && name !== "scrape") {
+ stderr.push(
+ `firecrawl ${name} isn't in this hosted shell, which has search and scrape. ${hint}`,
+ );
+ return result("", 2);
+ }
+ const c: Context = {
+ client,
+ fs: ctx.fs,
+ cwd: ctx.cwd,
+ signal: ctx.signal,
+ stderr,
+ };
+ try {
+ return result(await (name === "search" ? search : scrape)(c, rest), 0);
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ if (error instanceof Unsupported) {
+ stderr.push(`Error: ${message}. ${hint}`);
+ return result("", 2);
+ }
+ stderr.push(`Error: ${message}`);
+ return result("", 1);
+ }
+ });
+}
diff --git a/apps/api/src/lib/hosted-bash/limits.ts b/apps/api/src/lib/hosted-bash/limits.ts
new file mode 100644
index 000000000..759eb9acc
--- /dev/null
+++ b/apps/api/src/lib/hosted-bash/limits.ts
@@ -0,0 +1,3 @@
+export const TIMEOUT_MS = 30_000;
+export const MAX_OUTPUT_BYTES = 256_000;
+export const MAX_FIRECRAWL_CALLS = 25;
diff --git a/apps/api/src/lib/hosted-bash/shell-worker.ts b/apps/api/src/lib/hosted-bash/shell-worker.ts
new file mode 100644
index 000000000..934058073
--- /dev/null
+++ b/apps/api/src/lib/hosted-bash/shell-worker.ts
@@ -0,0 +1,188 @@
+import { Bash, type CommandName, type InitialFiles } from "just-bash";
+import {
+ FIRECRAWL_HELP,
+ FirecrawlClient,
+ firecrawlCommand,
+ type FirecrawlCaller,
+} from "./firecrawl-command";
+import { MAX_OUTPUT_BYTES } from "./limits";
+
+/** What the API process hands the shell process: a command, its caller and the docs pages to mount. */
+export interface ShellJob {
+ command: string;
+ caller: FirecrawlCaller;
+ docs: string[];
+ deadline: number;
+}
+
+export interface SandboxResult {
+ stdout: string;
+ stderr: string;
+ exitCode: number;
+ truncated: boolean;
+ /** Firecrawl endpoints the command called, and the ones that failed. */
+ endpoints: string[];
+ failedEndpoints: string[];
+}
+
+export type WorkerMessage =
+ | { type: "doc"; id: number; path: string }
+ | { type: "done"; result: SandboxResult };
+
+export type ParentMessage =
+ | { type: "job"; job: ShellJob }
+ | { type: "doc"; id: number; text?: string; error?: string };
+
+const MIB = 1024 * 1024;
+
+// Text tools only: no archives, databases, Python or JavaScript. bash, sh, awk
+// and xargs still run commands, under the same execution limits.
+const COMMANDS: CommandName[] = [
+ "echo",
+ "cat",
+ "printf",
+ "ls",
+ "mkdir",
+ "touch",
+ "rm",
+ "cp",
+ "mv",
+ "pwd",
+ "head",
+ "tail",
+ "wc",
+ "stat",
+ "grep",
+ "fgrep",
+ "egrep",
+ "rg",
+ "sed",
+ "awk",
+ "sort",
+ "uniq",
+ "comm",
+ "cut",
+ "paste",
+ "tr",
+ "rev",
+ "nl",
+ "fold",
+ "column",
+ "join",
+ "tee",
+ "find",
+ "basename",
+ "dirname",
+ "tree",
+ "du",
+ "env",
+ "xargs",
+ "true",
+ "false",
+ "bash",
+ "sh",
+ "jq",
+ "base64",
+ "date",
+ "sleep",
+ "seq",
+ "expr",
+ "tac",
+ "which",
+ "help",
+];
+
+const README = `# Firecrawl bash
+
+A stateless virtual shell with a hosted subset of the Firecrawl CLI as \`firecrawl\`.
+Each request starts from this fresh filesystem; nothing persists between requests.
+
+ /README.md this file
+ /docs/ the live Firecrawl docs (docs.firecrawl.dev) as markdown, fetched when read.
+ Start with /docs/llms.txt, or grep -l -r <term> /docs/features /docs/sdks
+
+${FIRECRAWL_HELP}`;
+
+function truncate(text: string): [string, boolean] {
+ const bytes = Buffer.from(text, "utf8");
+ if (bytes.length <= MAX_OUTPUT_BYTES) return [text, false];
+ const kept = bytes.subarray(0, MAX_OUTPUT_BYTES).toString("utf8");
+ return [`${kept}\n[truncated at ${MAX_OUTPUT_BYTES} bytes]\n`, true];
+}
+
+const send = (message: WorkerMessage) => process.send?.(message);
+
+// Docs pages are read through the API process, which caches them for every shell.
+const pending = new Map<
+ number,
+ { resolve: (text: string) => void; reject: (error: Error) => void }
+>();
+let nextId = 0;
+process.on("message", (message: ParentMessage) => {
+ if (message.type === "job") {
+ void run(message.job)
+ .catch(
+ (error: unknown): SandboxResult => ({
+ stdout: "",
+ stderr: `bash: ${error instanceof Error ? error.message : String(error)}\n`,
+ exitCode: 1,
+ truncated: false,
+ endpoints: [],
+ failedEndpoints: [],
+ }),
+ )
+ .then(result => send({ type: "done", result }));
+ return;
+ }
+ const waiter = pending.get(message.id);
+ if (!waiter) return;
+ pending.delete(message.id);
+ if (message.error === undefined) waiter.resolve(message.text ?? "");
+ else waiter.reject(new Error(message.error));
+});
+const readDoc = (path: string) =>
+ new Promise<string>((resolve, reject) => {
+ const id = nextId++;
+ pending.set(id, { resolve, reject });
+ send({ type: "doc", id, path });
+ });
+
+async function run(job: ShellJob): Promise<SandboxResult> {
+ const client = new FirecrawlClient(job.caller);
+ const files: InitialFiles = { "/README.md": README };
+ for (const path of job.docs) files[path] = () => readDoc(path);
+ const bash = new Bash({
+ files,
+ cwd: "/",
+ env: { HOME: "/", PWD: "/" },
+ commands: COMMANDS,
+ customCommands: [firecrawlCommand(client)],
+ executionLimitProfile: "hardened",
+ // The API process also kills this process at the deadline, since a single
+ // long command never reaches just-bash's own check.
+ executionLimits: {
+ maxExecutionTimeMs: Math.max(1, job.deadline - Date.now()),
+ maxFileSystemBytes: 32 * MIB,
+ maxStringLength: MIB,
+ maxHeredocSize: MIB,
+ maxOutputSize: 2 * MIB,
+ maxLiveBytes: 64 * MIB,
+ maxInputBytes: 32 * MIB,
+ },
+ });
+ const outcome = await bash.exec(job.command).catch((error: unknown) => ({
+ stdout: "",
+ stderr: `${error instanceof Error ? error.message : String(error)}\n`,
+ exitCode: 1,
+ }));
+ const [stdout, stdoutCut] = truncate(outcome.stdout);
+ const [stderr, stderrCut] = truncate(outcome.stderr);
+ return {
+ stdout,
+ stderr,
+ exitCode: outcome.exitCode,
+ truncated: stdoutCut || stderrCut,
+ endpoints: client.endpoints,
+ failedEndpoints: client.failedEndpoints,
+ };
+}
diff --git a/apps/api/src/lib/hosted-bash/shell.ts b/apps/api/src/lib/hosted-bash/shell.ts
new file mode 100644
index 000000000..6b5c48f99
--- /dev/null
+++ b/apps/api/src/lib/hosted-bash/shell.ts
@@ -0,0 +1,155 @@
+import type { InitialFiles } from "just-bash";
+import { fork } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { extname, join } from "node:path";
+import { docsFiles } from "./docs";
+import type { FirecrawlCaller } from "./firecrawl-command";
+import { MAX_FIRECRAWL_CALLS, TIMEOUT_MS } from "./limits";
+import type {
+ ParentMessage,
+ SandboxResult,
+ ShellJob,
+ WorkerMessage,
+} from "./shell-worker";
+
+// Compiled, this is shell-worker.js; under tsx or vitest it is the .ts source.
+const WORKER_FILE = join(__dirname, `shell-worker${extname(__filename)}`);
+const MAX_SHELLS = 4;
+const HEAP_MB = 192;
+// Under tsx the transpiler alone holds about 200 MB.
+const MAX_RSS_KB = (WORKER_FILE.endsWith(".ts") ? 576 : 320) * 1024;
+const RSS_POLL_MS = 50;
+const KILL_GRACE_MS = 2_000;
+const DOCS_WAIT_MS = 5_000;
+
+let running = 0;
+
+/** Every shell slot on this server is taken. */
+export class ShellBusyError extends Error {}
+
+export interface ShellResult extends SandboxResult {
+ durationMs: number;
+}
+
+type Caller = Omit<FirecrawlCaller, "maxCalls" | "deadline">;
+
+/**
+ * Runs one command in a fresh shell in its own process, so a command that
+ * spins or runs out of memory only costs that process, killed at the deadline.
+ * A worker thread is not enough: one huge allocation aborts the whole process.
+ */
+export async function runShell(
+ command: string,
+ caller: Caller,
+ signal: AbortSignal,
+): Promise<ShellResult> {
+ if (running >= MAX_SHELLS) throw new ShellBusyError();
+ running++;
+ try {
+ const started = Date.now();
+ const deadline = started + TIMEOUT_MS;
+ const docs = await Promise.race([
+ docsFiles(),
+ new Promise<InitialFiles>(resolve =>
+ setTimeout(() => resolve({}), DOCS_WAIT_MS).unref(),
+ ),
+ new Promise<InitialFiles>(resolve => {
+ if (signal.aborted) resolve({});
+ signal.addEventListener("abort", () => resolve({}), { once: true });
+ }),
+ ]);
+ if (signal.aborted) {
+ return {
+ stdout: "",
+ stderr: "bash: the request was closed\n",
+ exitCode: 130,
+ truncated: false,
+ endpoints: [],
+ failedEndpoints: [],
+ durationMs: Date.now() - started,
+ };
+ }
+ const job: ShellJob = {
+ command,
+ caller: { ...caller, maxCalls: MAX_FIRECRAWL_CALLS, deadline },
+ docs: Object.keys(docs),
+ deadline,
+ };
+ return await new Promise<ShellResult>(resolve => {
+ const child = fork(WORKER_FILE, [], {
+ env: {},
+ execArgv: [
+ `--max-old-space-size=${HEAP_MB}`,
+ ...(WORKER_FILE.endsWith(".ts") ? ["--require", "tsx/cjs"] : []),
+ ],
+ stdio: ["ignore", "ignore", "ignore", "ipc"],
+ });
+ let settled = false;
+ const send = (message: ParentMessage) => {
+ if (!settled && child.connected) child.send(message);
+ };
+ const finish = (result: SandboxResult) => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ clearInterval(watch);
+ signal.removeEventListener("abort", onAbort);
+ child.kill("SIGKILL");
+ resolve({ ...result, durationMs: Date.now() - started });
+ };
+ const fail = (exitCode: number, message: string) =>
+ finish({
+ stdout: "",
+ stderr: `bash: ${message}\n`,
+ exitCode,
+ truncated: false,
+ endpoints: [],
+ failedEndpoints: [],
+ });
+ const timer = setTimeout(
+ () =>
+ fail(
+ 124,
+ `execution exceeded the ${TIMEOUT_MS / 1000} second deadline`,
+ ),
+ deadline + KILL_GRACE_MS - Date.now(),
+ );
+ const onAbort = () => fail(130, "the request was closed");
+ signal.addEventListener("abort", onAbort);
+ // The heap flag misses large strings, so resident memory is watched too.
+ // Linux only; elsewhere the read fails and the heap flag is the only cap.
+ const watch = setInterval(() => {
+ readFile(`/proc/${child.pid}/status`, "utf8").then(
+ status => {
+ const rss = Number(/VmRSS:\s+(\d+)/.exec(status)?.[1] ?? 0);
+ if (rss > MAX_RSS_KB) fail(1, "memory limit exceeded");
+ },
+ () => {},
+ );
+ }, RSS_POLL_MS);
+
+ child.on("message", (message: WorkerMessage) => {
+ if (message.type === "done") return finish(message.result);
+ const read = docs[message.path] as (() => Promise<string>) | undefined;
+ (read ? read() : Promise.reject(new Error("No such file"))).then(
+ text => send({ type: "doc", id: message.id, text }),
+ (error: Error) =>
+ send({ type: "doc", id: message.id, error: error.message }),
+ );
+ });
+ child.on("error", error => fail(1, error.message));
+ // V8 aborts the process when the command outgrows its heap.
+ child.on("exit", (code, killSignal) =>
+ fail(
+ 1,
+ killSignal === "SIGABRT" || code === 134
+ ? "memory limit exceeded"
+ : "the shell stopped unexpectedly",
+ ),
+ );
+ send({ type: "job", job });
+ });
+ } finally {
+ running--;
+ }
+}