Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): hosted bash for agents at /v2/bash (experiment, off by default) (#5664)
Benjamin Werner793fd57f133b
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 9a8757cbf..9f545f41b 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -127,6 +127,7 @@"jose": "^6.2.12","jsdom": "^29.1.1","jsonrepair": "^3.14.0",+ "just-bash": "3.6.0","koffi": "^2.9.0","langsmith": "^0.6.0","lodash": "^4.18.0",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex cfb628b13..12b5d6045 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -187,7 +187,7 @@ importers:version: 16.4.5drizzle-orm:specifier: 1.0.0-rc.3- version: 1.0.0-rc.3(@opentelemetry/[email protected])(@types/[email protected])([email protected])([email protected])+ version: 1.0.0-rc.3(@opentelemetry/[email protected])(@types/[email protected])([email protected])([email protected])([email protected])esbuild:specifier: ^0.28.1version: 0.28.1@@ -230,6 +230,9 @@ importers:jsonrepair:specifier: ^3.14.0version: 3.14.0+ just-bash:+ specifier: 3.6.0+ version: 3.6.0koffi:specifier: ^2.9.0version: 2.9.0@@ -398,7 +401,7 @@ importers:version: 1.5.0'@napi-rs/cli':specifier: ^3.7.3+ version: 3.7.3(@emnapi/[email protected])(@emnapi/[email protected])(@types/[email protected])([email protected])'@taplo/cli':specifier: ^0.7.0version: 0.7.0@@ -554,6 +557,9 @@ packages:'@asamuzakjp/[email protected]':resolution: {integrity: sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==}+ '@borewit/[email protected]':+ resolution: {integrity: sha512-49CycTz1aAZmvFcwkeFyhud1HUmRIB5Tabf7ALV+VA4Iu85MTfz2cUwzPagMFAdVBvrWdkCxG67I30tUwwpKsw==}+'@bramus/[email protected]':resolution: {integrity: sha512-ctxtJ/eA+t+6q2++vj5j7FYX3nRu311q1wfYH3xjlLOsczhlhxAg2FWNUXhpGvAw3BWo1xBcvOV6/YLc2r5FJw==}hasBin: true@@ -1046,6 +1052,10 @@ packages:'@mixmark-io/[email protected]':resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}+ '@mongodb-js/[email protected]':+ resolution: {integrity: sha512-mQ2s0pYYiav+tzCDR05Zptem8Ey2v8s11lri5RKGhTtL4COVCvVCk5vtyRYNT+9L8qSfyOqqefF9UtnW8mC5jA==}+ engines: {node: '>= 20.19.0'}+'@msgpackr-extract/[email protected]':resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==}cpu: [arm64]@@ -2018,6 +2028,13 @@ packages:resolution: {integrity: sha512-Ck3zFhQhIhi02Hl6T4ZmJsXdnJE+wXcJz5f8klxd4keRYgenMnip3JDPMGDRLbnC/2iGd8P0sBIQqI3KxfVjBg==}hasBin: true+ '@tokenizer/[email protected]':+ resolution: {integrity: sha512-2mAv+8pkG6GIZiF1kNg1jAjh27IDxEPKwdGul3snfztFerfPGI1LjDezZp3i7BElXompqEtPmoPx6c2wgtWsOA==}+ engines: {node: '>=18'}++ '@tokenizer/[email protected]':+ resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==}+'@tootallnate/[email protected]':resolution: {integrity: sha512-VyMVKRrpHTT8PnotUeV8L/mDaMwD5DaAKCFLP73zAqAtvF0FCqky+Ki7BYbFCYQmqFyTe9316Ed5zS70QUR9eg==}engines: {node: '>= 10'}@@ -2449,6 +2466,9 @@ packages:resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==}+ resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==}+resolution: {integrity: sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==}engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16}@@ -2473,6 +2493,9 @@ packages:resolution: {integrity: sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==}+ resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}+resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==}@@ -2513,6 +2536,9 @@ packages:resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==}engines: {node: '>= 6'}+ resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}+resolution: {integrity: sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==}engines: {node: '>=18'}@@ -2571,6 +2597,10 @@ packages:resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==}engines: {node: '>=20'}+ resolution: {integrity: sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-T0+barUSQRTUQASh8bx02dl+DhF54GtIDY13Y3m9oWTklKbb3Wv974meRpeZ3lp1JpLVECWWNHC4vaG2XHXouQ==}@@ -2687,6 +2717,14 @@ packages:resolution: {integrity: sha512-vWHQkXgg2cE3BB4rR0c+PYw9BCQzLoNIRMI7AunIkZst74tw/vpPp7VLoYtsR/cy1f55g3XDWUslfeCZjiZJyA==}engines: {node: '>=14.16'}+ resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}+ engines: {node: '>=10'}++ resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==}+ engines: {node: '>=4.0.0'}+resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==}engines: {node: '>=0.4.0'}@@ -2974,6 +3012,10 @@ packages:resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==}engines: {node: '>=18.0.0'}+ resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==}+ engines: {node: '>=6'}+resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==}engines: {node: '>=12.0.0'}@@ -3044,6 +3086,10 @@ packages:resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}engines: {node: ^12.20 || >= 14.13}+ resolution: {integrity: sha512-Ievi/yy8DS3ygGvT47PjSfdFoX+2isQueoYP1cntFW1JLYAuS4GD7NUPGg4zv2iZfV52uDyk5w5Z0TdpRS6Q1g==}+ engines: {node: '>=20'}+resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==}@@ -3102,6 +3148,9 @@ packages:resolution: {integrity: sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==}+ resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==}+resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}@@ -3164,6 +3213,9 @@ packages:resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==}+ resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==}+resolution: {integrity: sha512-A1vjIwC3qpdCso45Iw127gnp7sPhqgY8lUQhDrArIs+GbmfaaVPFuoyy7WyLT9EPZCRZ7j7aLdwUhQmHWesEow==}engines: {node: 18 || 20 || >=22}@@ -3220,6 +3272,10 @@ packages:resolution: {integrity: sha512-+CqsMbHPiSTdtSO14O51eMNlrp9N79gmeqmXeouJOhfucAedHw9noVe/n5uJk3tbKE6a+6ZCQg3RPhVhHByAIw==}engines: {node: '>=18'}+ resolution: {integrity: sha512-PSaNyJk8VJlUVEhMHPXT/Z/WgNrLBh43WaYQJiu1qvyCuPQA+kgFMNmAOxI3GU3uXrCkO/xRW11XjcswrksYOA==}+ engines: {node: '>=20.19'}+resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}engines: {node: '>= 0.4'}@@ -3303,6 +3359,13 @@ packages:resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==}+ resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==}++ resolution: {integrity: sha512-IBTdIkzZNOpqm7q3dRqJvMaldXjDHWkEDfrwGEQTs5eaQMWV+djAhR+wahyNNMAa+qpbDUhBMVt4ZKNwpPm7xQ==}+ engines: {node: ^20.17.0 || >=22.9.0}+resolution: {integrity: sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA==}engines: {node: '>=12.22.0'}@@ -3407,6 +3470,11 @@ packages:resolution: {integrity: sha512-tWPGKMZf/8UPim+fcW2EfcQ/d/7aKUrP6IECz9G3Tu6Q5dX0orSleqJ9z6sSw7qrQkjF8/Edo4DvsWBZ8H+HNg==}hasBin: true+ resolution: {integrity: sha512-2COjEgcHdqKXeQSiNfPAHmpwQx4S8wATaIwog83g2t645U5tom6FED4CBBdr3M+0ioX4xZ26eODw3/nD6RQ6xw==}+ engines: {node: '>=20.19'}+ hasBin: true+resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==}@@ -3636,10 +3704,17 @@ packages:resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==}engines: {node: '>=18'}+ resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}+ engines: {node: '>=10'}+resolution: {integrity: sha512-iny/40PEGkNJQ5/Pv6z3nm6FbRAby32GeZRjCUJk2WnNDbgPSTcdI+DDvLqsgBdD4cD5sMtMheqPPnehCCa8Nw==}engines: {node: 18 || 20 || >=22}+ resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}+resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==}engines: {node: '>=16 || 14 >=14.17'}@@ -3648,6 +3723,13 @@ packages:resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==}engines: {node: '>=16 || 14 >=14.17'}+ resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==}++ resolution: {integrity: sha512-t9VmxaqrmANnEOBhpSDI6HD192Ge48k8vmWqQQL7hSFEqHEYwZbbsu49+aKLWZeRvFs3j1pMhXOqqF4kPlvjkQ==}+ engines: {node: '>=18.0.0'}+resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==}@@ -3674,6 +3756,9 @@ packages:engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1}hasBin: true+ resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==}+resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==}engines: {node: '>= 0.6'}@@ -3682,6 +3767,10 @@ packages:resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==}engines: {node: '>= 0.6'}+ resolution: {integrity: sha512-bkb/RkuWCE64eWu1PCh3Fu2pGc9mKRs+kOE+FU2bGHRYlQa86/D321DSATV54AsCQMKz3/xSt9sRgraZTpqdhQ==}+ engines: {node: '>=10'}+resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==}deprecated: This package is no longer needed on supported Node.js versions. Node.js 16+ includes AbortController and AbortSignal as built-in globals. All Node.js versions requiring this polyfill are end-of-life. Use the built-in globals instead.@@ -3689,6 +3778,10 @@ packages:resolution: {integrity: sha512-ibPK3iA+vaY1eEjESkQkM0BbCqFOaZMiXRTtdB0u7b4djtY6JnsjvPdUHVMg6xQt3B8fpTTWHI9A+ADjM9frzg==}+ resolution: {integrity: sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==}+ engines: {node: ^18 || ^20 || >= 21}+resolution: {integrity: sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==}@@ -3721,6 +3814,11 @@ packages:resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==}hasBin: true+ resolution: {integrity: sha512-s0KzNOWwOJJgPG6wxg6cKohnAl9Wk/oW1KrQaVzJBjQwVcUGPQCzpR46Ximygjqj/3KhOrtJXnYMp/xYAXp75g==}+ engines: {node: '>=16.0.0'}+ hasBin: true+resolution: {integrity: sha512-TZKxPvItzai9kN9H/TkmCtx/ZN/hvr3vUycjlfmH0ootY9yFBzNOpiXAdIn1Iteqsvk4lQn6B5PTrt+n6h8k/w==}engines: {node: ^18.17.0 || >=20.5.0}@@ -3822,6 +3920,9 @@ packages:resolution: {integrity: sha512-zAMfbta9nPA8AKkA4WdmNAT8R8s6Dk0/xwPsqNnRidMFcLo1YkLuPDTgs2f64XdzS8vbLNnoJ+uUIkgbz7oRHA==}+ resolution: {integrity: sha512-qOR+UDWv6D3hMxnWWOk0ZpBdCkd+kw5Yh7aXOaNSQU8A+73EA2bhW7EiUix/hY0yulUi5FRvt7PpYSI2VDvY0g==}+resolution: {integrity: sha512-Oq4j8LAOPOcssanQkIjxosjATBIEJhCxMCxPhMu+Ci4wdNmAEdx0O+a7gzbR2PyKXgKPvRLIN5g224+dJAsKHA==}@@ -3943,6 +4044,12 @@ packages:resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==}engines: {node: '>=0.10.0'}+ resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==}+ engines: {node: '>=10'}+ deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available.+ hasBin: true+resolution: {integrity: sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==}engines: {node: '>=14'}@@ -4017,6 +4124,13 @@ packages:resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==}engines: {node: '>= 0.10'}+ resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==}+ hasBin: true++ resolution: {integrity: sha512-s/I5zEAo79SUK0Qw4dpZKpiMwbQ6Gz0KU2NRr7eaO4x/p2g7Vvmn3hdeXDg8VsaUjfj/ora+e9oi27LX/C9+mw==}+resolution: {integrity: sha512-qpt8EwugBWDw2cgE2W+/3oxC+KTez2uSVR8JU9Q36TXPAGCaozfQUs59v4j4GFpWTaw0i6hAZSvOmu1J0uOEUg==}engines: {node: ^18.17.0 || >=20.5.0}@@ -4110,6 +4224,15 @@ packages:resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==}engines: {node: '>= 18'}+ resolution: {integrity: sha512-KUurW05zgjtZ2su0cBURC1q2RG/Uq0uzV3qURDsXqiKh03p9IXABQK4Qem56cFMqjT8TdQ2iTWsePrMprQWIRA==}+ engines: {node: '>=20.19.0'}+ peerDependencies:+ typescript: '>=5.1.6 <7'+ peerDependenciesMeta:+ typescript:+ optional: true+resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==}@@ -4127,6 +4250,10 @@ packages:resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==}engines: {node: '>=v12.22.7'}+ resolution: {integrity: sha512-SMguiTnYrhpLdk3PwfzHeotrcwi8bNV4iemL9tx9poR/yeaMYwB9VzR1w7b57DuWpuqR8n6oZboi0hj3AxZxQg==}+ hasBin: true+resolution: {integrity: sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==}engines: {node: '>=10'}@@ -4183,6 +4310,12 @@ packages:resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==}engines: {node: '>=14'}+ resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==}++ resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==}+resolution: {integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==}@@ -4213,6 +4346,12 @@ packages:resolution: {integrity: sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==}+ resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==}++ resolution: {integrity: sha512-q/JOZuJfmVh8Zf8Eo+cMJvtxAWeWBqTfJ6AQYGXZmcaLGHMwhA2niH3fQYHrBtPeOss1ttzHec07K5+rHaMRMw==}+resolution: {integrity: sha512-FUTYXpLroqytNKWIfHzlDWoy9E4tmBB/RklNMy6w3VJs+/XEYAHgbiylg4SS43iOk/9bM0BlJ2EDpFAGT66IoQ==}engines: {node: '>=4'}@@ -4284,6 +4423,10 @@ packages:resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==}engines: {node: '>=12'}+ resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==}+ engines: {node: '>=0.10.0'}+resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==}engines: {node: '>=14.16'}@@ -4295,6 +4438,10 @@ packages:resolution: {integrity: sha512-UM4nTLWzQiDBseVxUGxN0wJzIk0UU89Qi7pKXz9qISo+lhviJikTuikBsK/oFRlpvYHvSdz0TVoLyQ+sEpYwxA==}+ resolution: {integrity: sha512-iAqrCpp7UXD2FZiHWhLXqdJ6CmNeVzRkkHAyyhhKMVtbVm2KtPX3D/bJNonvtcyLTHS4rwqP2pz9fsjK1K4z0Q==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==}@@ -4317,6 +4464,13 @@ packages:os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android]hasBin: true+ resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==}++ resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==}+ engines: {node: '>=6'}+resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==}@@ -4367,6 +4521,10 @@ packages:resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==}engines: {node: '>=0.6'}+ resolution: {integrity: sha512-s9euYHM//T3loJtbNUYdbAzsMlvKBeRS+XCLFy+UGosMQH19Hwjap5k19i4PPAEJJBsVIpMpVTxl4fIqCuNfUg==}+ engines: {node: '>=14.16'}+resolution: {integrity: sha512-Loo5UUvLD9ScZ6jh8beX1T6sO1w2/MpCRpEP7V280GKMVUQ0Jzar2U3UJPsrdbziLEMMhu3Ujnq//rhiFuIeag==}engines: {node: '>=6'}@@ -4404,9 +4562,16 @@ packages:engines: {node: '>=18.0.0'}hasBin: true+ resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==}+resolution: {integrity: sha512-eCZGBN4nNNqM9Owkv9HAtWRYfLA4h909E/WGAWWBpmB275ehNhZyk87/Tpvjbp0jjNl9XwCsbe6bm6CqFsgD+A==}+ resolution: {integrity: sha512-EKZvCCFHh8sWttpAMs10U6niZFJ2cpz8LkCPlylxoG4JcHF50J7EHnnBW3AY1qXtUMu7kDjCp7dnggB1sIbqEQ==}+ engines: {node: '>=18', npm: '>=9'}+resolution: {integrity: sha512-ZW/lVMRabETuYCd9O9ZvMhAh8GslSqaUjxmK/JLPCh6l73CvLBiuXswj/+7LdnWOgYsQ130FqLzFz5aGT4I3Ug==}@@ -4426,6 +4591,11 @@ packages:engines: {node: '>=14.17'}hasBin: true+ resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==}+ engines: {node: '>=14.17'}+ hasBin: true+resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==}engines: {node: '>=14.17'}@@ -4436,6 +4606,10 @@ packages:engines: {node: '>=16.20.0'}hasBin: true+ resolution: {integrity: sha512-8iAasVS4wUx0gPLjH8Xtz2PeDzTSiy8QiLGu2DT3X5GU+egFEQhpnbtkehbAwjvDcxIERmCYcysiODFmE3Ud0Q==}+ engines: {node: '>=18'}+resolution: {integrity: sha512-PnKvRX+Fbxl/usVlTr/31tXIOVAH+8OdC9qaxUQcvurwEnoc3Vk/ipi2Ce8IjMxzK9byfSGdsUBjqqJGc1o5lw==}engines: {node: '>=14'}@@ -4873,6 +5047,8 @@ snapshots:'@asamuzakjp/[email protected]': {}+ '@borewit/[email protected]': {}+'@bramus/[email protected]':dependencies:css-tree: 3.2.1@@ -5337,6 +5513,12 @@ snapshots:'@mixmark-io/[email protected]': {}+ '@mongodb-js/[email protected]':+ dependencies:+ node-addon-api: 8.9.2+ prebuild-install: 7.1.3+ optional: true+'@msgpackr-extract/[email protected]':optional: true@@ -5355,7 +5537,7 @@ snapshots:'@msgpackr-extract/[email protected]':optional: true- '@napi-rs/[email protected](@emnapi/[email protected])(@emnapi/[email protected])(@types/[email protected])':+ '@napi-rs/[email protected](@emnapi/[email protected])(@emnapi/[email protected])(@types/[email protected])([email protected])':dependencies:'@inquirer/prompts': 8.5.2(@types/[email protected])'@napi-rs/cross-toolchain': 1.0.3(@emnapi/[email protected])(@emnapi/[email protected])@@ -5363,7 +5545,7 @@ snapshots:'@octokit/rest': 22.0.1clipanion: 4.0.0-rc.4([email protected])colorette: 2.0.20- emnapi: 1.11.2+ emnapi: 1.11.2([email protected])es-toolkit: 1.49.0js-yaml: 4.3.2obug: 2.1.4@@ -6038,6 +6220,15 @@ snapshots:'@taplo/[email protected]': {}+ '@tokenizer/[email protected]':+ dependencies:+ debug: 4.4.3+ token-types: 6.2.0+ transitivePeerDependencies:+ - supports-color++ '@tokenizer/[email protected]': {}+'@tootallnate/[email protected]': {}'@tybys/[email protected]':@@ -6418,6 +6609,13 @@ snapshots:+ dependencies:+ buffer: 5.7.1+ inherits: 2.0.4+ readable-stream: 3.6.2+ optional: true+dependencies:bytes: 3.1.2@@ -6461,6 +6659,12 @@ snapshots:+ dependencies:+ base64-js: 1.5.1+ ieee754: 1.2.1+ optional: true+dependencies:base64-js: 1.5.1@@ -6519,6 +6723,9 @@ snapshots:parse5: 7.1.2parse5-htmlparser2-tree-adapter: 7.0.0+ optional: true+dependencies:restore-cursor: 5.1.0@@ -6577,6 +6784,8 @@ snapshots:+ [email protected]: {}+@@ -6678,6 +6887,14 @@ snapshots:+ dependencies:+ mimic-response: 3.1.0+ optional: true++ optional: true+@@ -6719,11 +6936,12 @@ snapshots:- [email protected](@opentelemetry/[email protected])(@types/[email protected])([email protected])([email protected]):+ [email protected](@opentelemetry/[email protected])(@types/[email protected])([email protected])([email protected])([email protected]):optionalDependencies:'@opentelemetry/api': 1.9.1'@types/pg': 8.15.5pg: 8.16.3+ sql.js: 1.16.0zod: 4.6.7@@ -6749,7 +6967,9 @@ snapshots:- [email protected]: {}+ optionalDependencies:+ node-addon-api: 8.9.2@@ -6849,6 +7069,9 @@ snapshots:+ optional: true+@@ -6947,6 +7170,15 @@ snapshots:node-domexception: 1.0.0web-streams-polyfill: 3.3.3+ dependencies:+ '@tokenizer/inflate': 0.4.1+ strtok3: 10.7.0+ token-types: 6.2.0+ uint8array-extras: 1.6.0+ transitivePeerDependencies:+ - supports-color+optional: true@@ -7011,6 +7243,9 @@ snapshots:+ optional: true+optional: true@@ -7108,6 +7343,9 @@ snapshots:dependencies:resolve-pkg-maps: 1.0.0+ optional: true+dependencies:minimatch: 11.0.3@@ -7211,6 +7449,11 @@ snapshots:transitivePeerDependencies:- supports-color+ dependencies:+ ipaddr.js: 2.2.0+ undici: 7.30.1+@@ -7306,6 +7549,11 @@ snapshots:+ optional: true++ [email protected]: {}+dependencies:'@ioredis/commands': 1.2.0@@ -7404,6 +7652,30 @@ snapshots:+ dependencies:+ diff: 8.0.3+ fast-xml-parser: 5.12.0+ file-type: 21.3.4+ guarded-fetch: 0.1.3+ ini: 6.0.0+ modern-tar: 0.7.7+ papaparse: 5.7.1+ re2js: 1.3.3+ run: 2.1.3([email protected])+ seek-bzip: 2.0.0+ smol-toml: 1.9.0+ sprintf-js: 1.1.3+ sql.js: 1.16.0+ turndown: 7.2.5+ typescript: 5.9.3+ yaml: 2.9.1+ optionalDependencies:+ '@mongodb-js/zstd': 7.0.0+ node-liblzma: 2.2.0+ transitivePeerDependencies:+ - supports-color+dependencies:buffer-equal-constant-time: 1.0.1@@ -7595,15 +7867,26 @@ snapshots:+ optional: true+dependencies:'@isaacs/nonrecursive-glob-match': 1.0.3brace-expansion: 5.0.12+ optional: true++ optional: true++ [email protected]: {}+@@ -7634,15 +7917,26 @@ snapshots:+ optional: true++ dependencies:+ semver: 7.8.5+ optional: true+optional: true+ optional: true+@@ -7666,6 +7960,12 @@ snapshots:+ dependencies:+ node-addon-api: 8.9.2+ node-gyp-build: 4.8.4+ optional: true+@@ -7799,6 +8099,8 @@ snapshots:+ [email protected]: {}+@@ -7905,6 +8207,22 @@ snapshots:dependencies:xtend: 4.0.2+ dependencies:+ detect-libc: 2.1.2+ expand-template: 2.0.3+ github-from-package: 0.0.0+ minimist: 1.2.8+ mkdirp-classic: 0.5.3+ napi-build-utils: 2.0.0+ node-abi: 3.101.0+ pump: 3.0.4+ rc: 1.2.8+ simple-get: 4.0.1+ tar-fs: 2.1.5+ tunnel-agent: 0.6.0+ optional: true+@@ -7991,6 +8309,16 @@ snapshots:iconv-lite: 0.7.3unpipe: 1.0.0+ dependencies:+ deep-extend: 0.6.0+ ini: 1.3.8+ minimist: 1.2.8+ strip-json-comments: 2.0.1+ optional: true++ [email protected]: {}+dependencies:json-parse-even-better-errors: 4.0.0@@ -8106,6 +8434,10 @@ snapshots:transitivePeerDependencies:- supports-color+ optionalDependencies:+ typescript: 5.9.3+@@ -8118,6 +8450,10 @@ snapshots:dependencies:xmlchars: 2.2.0+ dependencies:+ commander: 6.2.1+@@ -8189,6 +8525,16 @@ snapshots:+ optional: true++ dependencies:+ decompress-response: 6.0.0+ once: 1.4.0+ simple-concat: 1.0.1+ optional: true+dependencies:is-arrayish: 0.3.2@@ -8215,6 +8561,10 @@ snapshots:dependencies:through: 2.3.8+ [email protected]: {}++ [email protected]: {}+dependencies:bindings: 1.5.0@@ -8285,6 +8635,9 @@ snapshots:dependencies:ansi-regex: 6.4.0+ optional: true+@@ -8296,6 +8649,10 @@ snapshots:dependencies:anynum: 1.0.1+ dependencies:+ '@tokenizer/token': 0.3.0+@@ -8324,6 +8681,23 @@ snapshots:+ dependencies:+ chownr: 1.1.4+ mkdirp-classic: 0.5.3+ pump: 3.0.4+ tar-stream: 2.2.0+ optional: true++ dependencies:+ bl: 4.1.0+ end-of-stream: 1.4.5+ fs-constants: 1.0.0+ inherits: 2.0.4+ readable-stream: 3.6.2+ optional: true+dependencies:bintrees: 1.0.2@@ -8377,6 +8751,12 @@ snapshots:+ dependencies:+ '@borewit/text-codec': 0.2.3+ '@tokenizer/token': 0.3.0+ ieee754: 1.2.1+dependencies:psl: 1.15.0@@ -8415,10 +8795,19 @@ snapshots:optionalDependencies:fsevents: 2.3.3+ dependencies:+ safe-buffer: 5.2.1+ optional: true+dependencies:'@mixmark-io/domino': 2.2.0+ dependencies:+ '@mixmark-io/domino': 2.2.0+@@ -8436,6 +8825,8 @@ snapshots:+ [email protected]: {}+@@ -8461,6 +8852,8 @@ snapshots:'@typescript/typescript-win32-arm64': 7.0.2'@typescript/typescript-win32-x64': 7.0.2+ [email protected]: {}+diff --git a/apps/api/pnpm-workspace.yaml b/apps/api/pnpm-workspace.yamlindex 112612a2a..f01831083 100644--- a/apps/api/pnpm-workspace.yaml+++ b/apps/api/pnpm-workspace.yaml@@ -122,6 +122,10 @@ allowBuilds:# @google-cloud/bigtable); the pure-JS fallback is fine for our payload# sizes, so no native compilation.sse4_crc32: false+ # Optional native compressors for just-bash's archive commands, which the+ # hosted shell (/v2/bash) does not expose.+ "@mongodb-js/zstd": false+ node-liblzma: falsebufferutil: trueesbuild: truefoundationdb: truediff --git a/apps/api/src/__tests__/snips/v2/bash.test.ts b/apps/api/src/__tests__/snips/v2/bash.test.tsnew file mode 100644index 000000000..8a48e32b7--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/bash.test.ts@@ -0,0 +1,53 @@+import { ALLOW_TEST_SUITE_WEBSITE, itIf, TEST_SUITE_WEBSITE } from "../lib";+import request, { idmux, Identity, scrapeTimeout, TEST_API_URL } from "./lib";++describe("Hosted bash", () => {+ let identity: Identity;++ beforeAll(async () => {+ identity = await idmux({ name: "bash", concurrency: 10, credits: 1000 });+ }, 10000);++ const run = (command: string) =>+ request(TEST_API_URL)+ .post("/v2/bash")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .type("form")+ .send(command);++ it("serves usage", async () => {+ const res = await request(TEST_API_URL).get("/v2/bash");+ expect(res.status).toBe(200);+ expect(res.text).toContain("/v2/bash --data-binary 'firecrawl search");+ });++ it("runs a command in a fresh shell", async () => {+ const res = await run("echo hello | tr a-z A-Z; cat /README.md | head -1");+ expect(res.status).toBe(200);+ expect(res.headers["x-exit-code"]).toBe("0");+ expect(res.text).toBe("HELLO\n# Firecrawl bash\n");+ });++ itIf(ALLOW_TEST_SUITE_WEBSITE)(+ "scrapes through the API's own scrape route",+ async () => {+ const res = await run(+ `firecrawl scrape ${TEST_SUITE_WEBSITE} | grep -c Firecrawl`,+ );+ expect(res.headers["x-exit-code"]).toBe("0");+ expect(Number(res.text.trim())).toBeGreaterThan(0);+ },+ scrapeTimeout,+ );++ it("exits 2 for a command outside the hosted subset", async () => {+ const res = await run("firecrawl crawl https://example.com");+ expect(res.headers["x-exit-code"]).toBe("2");+ expect(res.text).toContain("npx -y firecrawl-cli@latest init");+ });++ it("stops a runaway command", async () => {+ const res = await run("while true; do :; done");+ expect(res.headers["x-exit-code"]).toBe("126");+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/bash.test.ts b/apps/api/src/controllers/v2/__tests__/bash.test.tsnew file mode 100644index 000000000..bff86375d--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/bash.test.ts@@ -0,0 +1,348 @@+import express from "express";+import type { Server } from "http";+import type { AddressInfo } from "net";+import request from "supertest";+import { registerHostedBashRoute } from "../bash";++const DOCS_INDEX = `# Firecrawl Docs+- [Intro](https://docs.firecrawl.dev/introduction.md): Start here.+- [CLI](https://docs.firecrawl.dev/sdks/cli.md): The CLI.+`;++interface InnerCall {+ path: string;+ ip: string | undefined;+ authorization: string | undefined;+ body: any;+}++/**+ * The route under test plus stand-ins for /v2/search and /v2/scrape on the same+ * server, since the shell calls back into the server that answered it.+ */+describe("hosted bash route", () => {+ const realFetch = globalThis.fetch;+ const calls: InnerCall[] = [];+ let respond: (call: InnerCall) => { status: number; json: unknown };+ let server: Server;+ let base: string;++ beforeAll(async () => {+ const app = express();+ // One trusted hop, like the load balancer in front of the real API.+ app.set("trust proxy", 1);+ registerHostedBashRoute(app);+ app.use(express.json());+ app.post(["/v2/search", "/v2/scrape"], (req, res) => {+ const call: InnerCall = {+ path: req.path,+ ip: req.ip,+ authorization: req.get("authorization"),+ body: req.body,+ };+ calls.push(call);+ const { status, json } = respond(call);+ res.status(status).json(json);+ });+ server = app.listen(0, "127.0.0.1");+ await new Promise(resolve => server.once("listening", resolve));+ base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;++ // The docs mount reads docs.firecrawl.dev; everything else is the real fetch.+ vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {+ const url = String(input);+ if (!url.startsWith("https://docs.firecrawl.dev/")) {+ return realFetch(input, init);+ }+ const body = url.endsWith("/llms.txt") ? DOCS_INDEX : `page ${url}\n`;+ return Promise.resolve(new Response(body));+ });+ });++ afterAll(async () => {+ vi.restoreAllMocks();+ await new Promise(resolve => server.close(resolve));+ });++ beforeEach(() => {+ calls.length = 0;+ respond = call =>+ call.path === "/v2/scrape"+ ? {+ status: 200,+ json: {+ success: true,+ data: {+ markdown: "# Example",+ links: ["https://a.test", "https://b.test"],+ metadata: { title: "Example" },+ },+ },+ }+ : {+ status: 200,+ json: {+ success: true,+ id: "search-1",+ creditsUsed: 2,+ data: {+ web: [+ {+ title: "First",+ url: "https://first.test",+ description: "About first",+ },+ { url: "https://second.test", markdown: "line 1\nline 2" },+ ],+ },+ },+ };+ });++ // The caller's address as the load balancer would report it.+ const run = (command: string, ip = "203.0.113.9") =>+ request(base)+ .post("/v2/bash")+ .set("X-Forwarded-For", ip)+ .type("form")+ .send(command);++ it("serves usage that points at its own URL", async () => {+ const res = await request(base).get("/v2/bash");+ expect(res.status).toBe(200);+ expect(res.text).toContain(+ `curl ${base}/v2/bash --data-binary 'firecrawl search`,+ );+ });++ it("runs a raw command sent the way curl --data-binary sends it", async () => {+ const res = await run("echo 'a=1&b=2' | tr a-z A-Z; ls /");+ expect(res.status).toBe(200);+ expect(res.headers["x-exit-code"]).toBe("0");+ expect(res.text).toBe("A=1&B=2\nREADME.md\nbin\ndev\ndocs\nproc\nusr\n");+ });++ it("takes a command from the query string or a JSON body, and returns JSON", async () => {+ const get = await request(base)+ .get("/v2/bash")+ .query({ command: "echo from-query" });+ expect(get.text).toBe("from-query\n");++ const post = await request(base)+ .post("/v2/bash?format=json")+ .send({ command: "echo out; echo err >&2; exit 3" });+ expect(post.body).toMatchObject({+ stdout: "out\n",+ stderr: "err\n",+ exitCode: 3,+ truncated: false,+ });+ expect(post.headers["x-exit-code"]).toBe("3");+ });++ it("mounts the docs and reads pages on demand", async () => {+ const res = await run("ls /docs /docs/sdks; cat /docs/sdks/cli.md");+ expect(res.text).toBe(+ "/docs:\nintroduction.md\nllms.txt\nsdks\n\n/docs/sdks:\ncli.md\npage https://docs.firecrawl.dev/sdks/cli.md\n",+ );+ });++ it("scrapes through the API's own route as the caller", async () => {+ const res = await run("firecrawl scrape example.com --full-page");+ expect(res.text).toBe("# Example\n");+ expect(calls).toHaveLength(1);+ expect(calls[0]).toMatchObject({+ path: "/v2/scrape",+ // Auth and quotas key on this: it must be the caller, not loopback.+ ip: "203.0.113.9",+ authorization: undefined,+ body: {+ url: "https://example.com",+ formats: ["markdown"],+ onlyMainContent: false,+ origin: "bash",+ integration: "_firecrawl-bash",+ },+ });+ });++ it("gives each caller its own identity", async () => {+ await run("firecrawl scrape https://example.com", "198.51.100.1");+ await run("firecrawl scrape https://example.com", "198.51.100.2");+ expect(calls.map(call => call.ip)).toEqual([+ "198.51.100.1",+ "198.51.100.2",+ ]);+ });++ it("passes the caller's API key through", async () => {+ await run("firecrawl scrape https://example.com").set(+ "Authorization",+ "Bearer fc-test",+ );+ expect(calls[0].authorization).toBe("Bearer fc-test");+ });++ it("prints scrape formats the way the CLI does", async () => {+ const res = await run(+ [+ "firecrawl scrape https://example.com links",+ "firecrawl scrape https://example.com -f markdown,links",+ "firecrawl scrape https://example.com -o out/page.md && cat out/page.md",+ ].join("\n"),+ );+ expect(res.text).toBe(+ [+ "https://a.test\nhttps://b.test",+ '{"markdown":"# Example","links":["https://a.test","https://b.test"],"metadata":{"title":"Example"}}',+ "# Example",+ ].join("\n"),+ );+ });++ it("saves several URLs to .firecrawl/ and reports the ones that fail", async () => {+ respond = call =>+ call.body.url.includes("bad")+ ? { status: 500, json: { success: false, error: "boom" } }+ : { status: 200, json: { success: true, data: { markdown: "ok" } } };+ const res = await run(+ "firecrawl scrape https://good.test/a/b https://bad.test; ls .firecrawl",+ );+ expect(res.text).toContain("good.test-a-b.md\n");+ expect(res.text).toContain("Scraping 2 URLs...");+ expect(res.text).toContain("Saved: .firecrawl/good.test-a-b.md");+ expect(res.text).toContain("Error: https://bad.test - boom");+ expect(res.text).toContain("Completed: 1/2 succeeded, 1 failed");+ expect(res.headers["x-exit-code"]).toBe("0");+ });++ it("searches, in the CLI's readable and JSON output", async () => {+ const readable = await run('firecrawl search "two words" --limit 2');+ expect(readable.text).toBe(+ [+ "First",+ " URL: https://first.test",+ " About first",+ "",+ "Untitled",+ " URL: https://second.test",+ "",+ " --- Content ---",+ " line 1",+ " line 2",+ " --- End Content ---",+ "",+ ].join("\n"),+ );+ expect(calls[0].body).toMatchObject({+ query: "two words",+ limit: 2,+ sources: [{ type: "web" }],+ });++ const json = await run(+ "firecrawl search x --json | jq -r '.id, .creditsUsed, .data.web[0].url'",+ );+ expect(json.text).toBe("search-1\n2\nhttps://first.test\n");+ });++ it("treats words and anything after -- as the query, not as options", async () => {+ await run("firecrawl search help desk; firecrawl search -- --limit -h");+ expect(calls.map(call => call.body.query)).toEqual([+ "help desk",+ "--limit -h",+ ]);+ });++ it("rejects a JSON body without a command", async () => {+ for (const body of ['{"cmd": "echo hi"}', "{not json"]) {+ const res = await request(base).post("/v2/bash").type("json").send(body);+ expect(res.status).toBe(400);+ expect(res.text).toContain('{"command": "..."}');+ }+ });++ it("shows the API's own error, such as a keyless limit, and exits 1", async () => {+ respond = () => ({+ status: 429,+ json: { success: false, error: "You've hit the keyless limit." },+ });+ const res = await run("firecrawl search anything");+ expect(res.text).toBe("Error: You've hit the keyless limit.\n[exit 1]\n");+ expect(res.headers["x-exit-code"]).toBe("1");+ });++ it("explains a bare keyless refusal", async () => {+ respond = () => ({+ status: 401,+ json: { success: false, error: "Unauthorized" },+ });+ const res = await run("firecrawl scrape https://example.com");+ expect(res.text).toContain("Create a free API key at");+ expect(res.headers["x-exit-code"]).toBe("1");+ });++ it("exits 2 without calling the API for anything outside the subset", async () => {+ const res = await run(+ [+ "firecrawl crawl https://example.com",+ "firecrawl scrape https://example.com --screenshot",+ "firecrawl scrape https://example.com -f screenshot",+ "firecrawl search x --sources alexandria",+ ].join("; echo $?; "),+ );+ expect(res.text.match(/^2$/gm)).toHaveLength(3);+ expect(res.text.match(/npx -y firecrawl-cli@latest init/g)).toHaveLength(4);+ expect(res.headers["x-exit-code"]).toBe("2");+ expect(calls).toHaveLength(0);+ });++ it("stops runaway commands and leaves out everything but text tools", async () => {+ const loop = await run("while true; do :; done");+ expect(loop.headers["x-exit-code"]).toBe("126");+ expect(Number(loop.headers["x-duration-ms"])).toBeLessThan(5000);++ const memory = await run(+ 's=xxxxxxxx; for i in $(seq 1 22); do s="$s$s"; done; echo ${#s}',+ );+ expect(memory.headers["x-exit-code"]).toBe("126");++ const tools = await run(+ "sqlite3 :memory: 'select 1'; curl https://example.com; python3 -c 1",+ );+ expect(tools.headers["x-exit-code"]).toBe("127");+ });++ it("caps Firecrawl calls per command", async () => {+ const res = await run(+ "for i in $(seq 1 30); do firecrawl scrape https://example.com > /dev/null || break; done",+ );+ expect(calls).toHaveLength(25);+ expect(res.text).toContain("call limit reached (25 Firecrawl calls");+ });++ it("truncates large output", async () => {+ const res = await run(+ 's=xxxxxxxx; for i in $(seq 1 17); do s="$s$s"; done; echo "$s"',+ );+ expect(res.text).toContain("[truncated at 256000 bytes]");+ expect(res.text.length).toBeLessThan(260_000);+ });++ it("rejects an oversized command", async () => {+ const res = await run(`echo ${"x".repeat(70_000)}`);+ expect(res.status).toBe(413);+ });++ it("sheds a caller that sends too many commands", async () => {+ const statuses: number[] = [];+ for (let i = 0; i < 62; i++) {+ const res = await request(base)+ .get("/v2/bash")+ .set("X-Forwarded-For", "192.0.2.77");+ statuses.push(res.status);+ }+ expect(statuses.slice(0, 60).every(status => status === 200)).toBe(true);+ expect(statuses.slice(60)).toEqual([429, 429]);+ });+});diff --git a/apps/api/src/controllers/v2/bash.ts b/apps/api/src/controllers/v2/bash.tsnew file mode 100644index 000000000..f25d0318f--- /dev/null+++ b/apps/api/src/controllers/v2/bash.ts@@ -0,0 +1,311 @@+import cors from "cors";+import express, {+ type Application,+ type Request,+ type RequestHandler,+ type Response,+} from "express";+import { isIPv4 } from "net";+import {+ RateLimiterMemory,+ RateLimiterRedis,+ RateLimiterRes,+} from "rate-limiter-flexible";+import { config } from "../../config";+import {+ MAX_FIRECRAWL_CALLS,+ MAX_OUTPUT_BYTES,+ TIMEOUT_MS,+} from "../../lib/hosted-bash/limits";+import {+ ShellBusyError,+ runShell,+ type ShellResult,+} from "../../lib/hosted-bash/shell";+import { keylessTeamId, keylessTeamUuid } from "../../lib/keyless-team";+import { logger } from "../../lib/logger";+import { capturePostHog } from "../../services/posthog-capture";+import { redisRateLimitClient } from "../../services/rate-limiter";++const ROUTE = "/v2/bash";+const RUN_EVENT = "bash_command_run";+const USAGE_EVENT = "bash_usage_viewed";+const MAX_COMMAND_BYTES = 64 * 1024;+const COMMANDS_PER_MINUTE = 60;+// CLI commands worth counting by name; any other word after `firecrawl` is "other".+const CLI_COMMANDS = new Set([+ "search",+ "scrape",+ "crawl",+ "map",+ "extract",+ "agent",+ "parse",+ "download",+ "batch-scrape",+ "interact",+ "browser",+ "monitor",+ "login",+ "init",+ "help",+]);++function usage(base: string): string {+ return `Firecrawl Bash++A hosted shell for agents. Send one bash command over HTTP and get its output back.+A subset of the Firecrawl CLI (search and scrape) is built in as \`firecrawl\`, and the+live Firecrawl docs are mounted at /docs. No API key, MCP server or install needed.++Run a command:+ curl ${base} --data-binary 'firecrawl search "stripe pricing" --limit 3'+ curl ${base} --data-binary 'firecrawl scrape https://stripe.com/pricing | head -40'+ curl ${base} --data-binary 'firecrawl --help'+ GET ${base}?command=firecrawl%20search%20stripe%20pricing+ POST ${base} JSON: {"command": "..."}++Read the Firecrawl docs:+ curl ${base} --data-binary 'cat /docs/llms.txt | head -40'+ curl ${base} --data-binary 'grep -l -r keyless /docs/features'++Pipes, grep, jq, sed, awk and loops work, for example:+ curl ${base} --data-binary 'firecrawl search "firecrawl changelog" --limit 5 --json | jq -r ".data.web[].url"'++Response:+ Plain text (stdout, then stderr) by default. Add ?format=json or Accept: application/json+ for stdout, stderr, exitCode, durationMs and truncated. X-Exit-Code is set on every run.++Without an API key, firecrawl runs on the free keyless tier, limited per IP per day.+A limit message includes a link to a free API key; send one as Authorization: Bearer fc-...+for higher limits. For crawl, map and everything else, install the Firecrawl CLI:+npx -y firecrawl-cli@latest init++Limits: ${MAX_COMMAND_BYTES} byte commands, ${MAX_OUTPUT_BYTES} bytes of stdout and stderr each,+${TIMEOUT_MS / 1000} seconds and ${MAX_FIRECRAWL_CALLS} Firecrawl calls per command. Nothing persists between requests.+`;+}++/**+ * Commands per IP per minute across every replica. Firecrawl calls also count+ * against the keyless tier or the caller's plan; commands that make none have+ * only this limit.+ */+function createRateLimit(): RequestHandler {+ const memory = new RateLimiterMemory({+ points: COMMANDS_PER_MINUTE,+ duration: 60,+ });+ const limiter = config.REDIS_RATE_LIMIT_URL+ ? new RateLimiterRedis({+ storeClient: redisRateLimitClient,+ keyPrefix: "hosted-bash",+ points: COMMANDS_PER_MINUTE,+ duration: 60,+ insuranceLimiter: memory,+ })+ : memory;+ return async (req, res, next) => {+ try {+ await limiter.consume(req.ip ?? "unknown");+ next();+ } catch (rejection) {+ if (!(rejection instanceof RateLimiterRes)) return next(rejection);+ const seconds = Math.max(1, Math.ceil(rejection.msBeforeNext / 1000));+ res.setHeader("Retry-After", String(seconds));+ res+ .status(429)+ .type("text/plain")+ .send(`Too many commands. Try again in ${seconds} seconds.\n`);+ }+ };+}++/** The command sent, undefined when none was, or null for a JSON body without one. */+function readCommand(req: Request): string | null | undefined {+ const fromQuery =+ typeof req.query.command === "string" ? req.query.command : undefined;+ const body = typeof req.body === "string" ? req.body : "";+ if (!body.trim()) return fromQuery;+ const json = req.is("application/json");+ // A JSON object is a request even without its content type; a brace group+ // like `{ echo a; }` does not parse, so it stays a command.+ if (json || body.trimStart().startsWith("{")) {+ try {+ const parsed = JSON.parse(body);+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {+ return typeof parsed.command === "string" ? parsed.command : null;+ }+ } catch {}+ if (json) return null;+ }+ // curl --data-urlencode 'command=...' sends a form; --data-binary also labels+ // a raw command as a form, so only the command= field is decoded.+ if (+ req.is("application/x-www-form-urlencoded") &&+ body.startsWith("command=")+ ) {+ return new URLSearchParams(body).get("command") ?? "";+ }+ return body;+}++/** Where this server answered the request, so the shell can call its routes. */+function ownBaseUrl(req: Request): string {+ const { localAddress, localPort } = req.socket;+ const host = !localAddress+ ? "127.0.0.1"+ : localAddress.startsWith("::ffff:")+ ? localAddress.slice(7)+ : localAddress.includes(":")+ ? `[${localAddress}]`+ : localAddress;+ return `http://${host}:${localPort}`;+}++/** Records a usage view, or the shape of a run (subcommands, outcome), never the command text. */+function track(+ req: Request,+ ownKey: boolean,+ run?: { command: string; result: ShellResult },+) {+ const ip = (req.ip ?? "").replace(/^::ffff:/i, "");+ // The id keyless_prompt_shown uses, so keyless runs join to prompts and signups.+ const keylessTeam =+ !ownKey && isIPv4(ip) ? keylessTeamUuid(keylessTeamId(ip)) : null;+ const event = run ? RUN_EVENT : USAGE_EVENT;+ const properties = {+ keyless_team_id: keylessTeam,+ user_agent: req.get("user-agent")?.slice(0, 300) ?? null,+ method: req.method,+ own_key: ownKey,+ ...(run+ ? {+ subcommands: [+ ...new Set(+ [...run.command.matchAll(/\bfirecrawl\s+([a-z][\w-]*)/g)].map(+ match => (CLI_COMMANDS.has(match[1]) ? match[1] : "other"),+ ),+ ),+ ],+ exit_code: run.result.exitCode,+ duration_ms: run.result.durationMs,+ endpoints: run.result.endpoints,+ failed_endpoints: run.result.failedEndpoints,+ truncated: run.result.truncated,+ }+ : {}),+ };+ logger.info("Hosted bash request", {+ module: "hosted-bash",+ event,+ ...properties,+ });+ void capturePostHog(event, keylessTeam ?? "hosted-bash-anonymous", {+ ...properties,+ // Callers are anonymous IPs: do not create a person for each one.+ $process_person_profile: false,+ });+}++async function bashController(req: Request, res: Response) {+ res.setHeader("Cache-Control", "no-store");+ const sent = req.method === "HEAD" ? undefined : readCommand(req);+ const authorization = req.get("authorization")?.trim() || null;++ if (sent === null) {+ res+ .status(400)+ .type("text/plain")+ .send('Send JSON as {"command": "..."}, or the command as the body.\n');+ return;+ }+ const command = sent?.trim();+ if (!command) {+ track(req, Boolean(authorization));+ res+ .type("text/plain")+ .send(usage(`${req.protocol}://${req.get("host")}${ROUTE}`));+ return;+ }+ if (Buffer.byteLength(command) > MAX_COMMAND_BYTES) {+ res+ .status(413)+ .type("text/plain")+ .send(`command is larger than ${MAX_COMMAND_BYTES} bytes\n`);+ return;+ }++ const userAgent = req.get("user-agent")?.slice(0, 500);+ const worldId = req.get("x-firecrawl-world-id");+ const closed = new AbortController();+ res.on("close", () => {+ if (!res.writableFinished) closed.abort();+ });+ let result: ShellResult;+ try {+ result = await runShell(+ command,+ {+ baseUrl: ownBaseUrl(req),+ ip: req.ip ?? null,+ authorization,+ headers: {+ ...(userAgent ? { "User-Agent": userAgent } : {}),+ ...(worldId ? { "X-Firecrawl-World-Id": worldId } : {}),+ },+ },+ closed.signal,+ );+ } catch (error) {+ if (!(error instanceof ShellBusyError)) throw error;+ res+ .status(503)+ .setHeader("Retry-After", "5")+ .type("text/plain")+ .send(+ "Every shell on this server is busy. Try again in a few seconds.\n",+ );+ return;+ }+ track(req, Boolean(authorization), { command, result });++ res.setHeader("X-Exit-Code", String(result.exitCode));+ res.setHeader("X-Duration-Ms", String(result.durationMs));+ if (req.query.format === "json" || req.accepts(["text", "json"]) === "json") {+ const { stdout, stderr, exitCode, durationMs, truncated } = result;+ res.json({ stdout, stderr, exitCode, durationMs, truncated });+ return;+ }+ let body = result.stdout;+ if (result.stderr) {+ body += (body && !body.endsWith("\n") ? "\n" : "") + result.stderr;+ }+ if (result.exitCode !== 0) {+ body += `${body && !body.endsWith("\n") ? "\n" : ""}[exit ${result.exitCode}]\n`;+ }+ res.type("text/plain").send(body);+}++/**+ * Hosted bash for agents: one command per request, in a fresh sandboxed shell.+ * Registered ahead of the global body parsers so the body is read once, as+ * text, under this route's own size limit.+ */+export function registerHostedBashRoute(+ app: Pick<Application, "get" | "post" | "options">,+) {+ const allowCors = cors({+ exposedHeaders: ["X-Exit-Code", "X-Duration-Ms", "Retry-After"],+ });+ const rateLimit = createRateLimit();+ app.options(ROUTE, allowCors);+ app.get(ROUTE, allowCors, rateLimit, bashController);+ app.post(+ ROUTE,+ allowCors,+ rateLimit,+ express.text({ type: () => true, limit: MAX_COMMAND_BYTES }),+ bashController,+ );+}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 87dfa9a31..da6ddcf79 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -34,6 +34,7 @@ import { v2Router } from "./routes/v2";import { exchangeRouter } from "./routes/exchange";import { labsRouter } from "./routes/labs";import { registerMcpActionLogIngestRoute } from "./routes/mcp-action-logs";+import { registerHostedBashRoute } from "./controllers/v2/bash";import { nuqShutdown } from "./services/worker/nuq";import { getErrorContactMessage } from "./lib/deployment";import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";@@ -91,6 +92,7 @@ const captureRawBody = (};registerMcpActionLogIngestRoute(app);+registerHostedBashRoute(app);app.use(bodyParser.urlencoded({ extended: true, verify: captureRawBody }));app.use(bodyParser.json({ limit: "10mb", verify: captureRawBody }));diff --git a/apps/api/src/lib/hosted-bash/docs.test.ts b/apps/api/src/lib/hosted-bash/docs.test.tsnew file mode 100644index 000000000..8edad7b40--- /dev/null+++ b/apps/api/src/lib/hosted-bash/docs.test.ts@@ -0,0 +1,85 @@+const ROOT = "https://docs.firecrawl.dev/llms.txt";+const SUB = "https://docs.firecrawl.dev/_llms/en/v2.txt";+const ROOT_INDEX = `- [Sub](${SUB}): more.+- [Intro](https://docs.firecrawl.dev/introduction.md): Start here.+`;+const SUB_INDEX = `- [CLI](https://docs.firecrawl.dev/sdks/cli.md): The CLI.+`;++describe("hosted bash docs mount", () => {+ let failing: Set<string>;+ let fetched: string[];++ // A fresh module per test: the page list is cached at module level.+ async function load() {+ vi.resetModules();+ return (await import("./docs.js")).docsFiles;+ }++ beforeEach(() => {+ failing = new Set();+ fetched = [];+ vi.spyOn(globalThis, "fetch").mockImplementation(input => {+ const url = String(input);+ fetched.push(url);+ if (failing.has(url)) return Promise.reject(new Error("docs down"));+ return Promise.resolve(+ new Response(url === ROOT ? ROOT_INDEX : SUB_INDEX),+ );+ });+ });++ afterEach(() => {+ vi.useRealTimers();+ vi.restoreAllMocks();+ });++ it("lists pages from the root index and its English sub-indexes", async () => {+ const docsFiles = await load();+ expect(Object.keys(await docsFiles()).sort()).toEqual([+ "/docs/introduction.md",+ "/docs/llms.txt",+ "/docs/sdks/cli.md",+ ]);+ });++ it("keeps the pages it found when one sub-index fails", async () => {+ failing.add(SUB);+ const docsFiles = await load();+ expect(Object.keys(await docsFiles()).sort()).toEqual([+ "/docs/introduction.md",+ "/docs/llms.txt",+ ]);+ });++ it("starts with an empty /docs when the docs site is down, and retries later", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ failing.add(ROOT);+ const docsFiles = await load();+ expect(Object.keys(await docsFiles())).toEqual(["/docs/llms.txt"]);++ // Within the retry window a command does not touch the docs site at all.+ fetched.length = 0;+ await docsFiles();+ expect(fetched).toEqual([]);++ failing.clear();+ vi.setSystemTime(Date.now() + 61_000);+ await docsFiles();+ await vi.waitFor(async () =>+ expect(Object.keys(await docsFiles())).toHaveLength(3),+ );+ });++ it("serves the last list while a later refresh fails", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ const docsFiles = await load();+ expect(Object.keys(await docsFiles())).toHaveLength(3);++ failing.add(ROOT).add(SUB);+ vi.setSystemTime(Date.now() + 2 * 60 * 60 * 1000);+ expect(Object.keys(await docsFiles())).toHaveLength(3);+ await new Promise(resolve => setImmediate(resolve));+ expect(Object.keys(await docsFiles())).toHaveLength(3);+ });+});diff --git a/apps/api/src/lib/hosted-bash/docs.ts b/apps/api/src/lib/hosted-bash/docs.tsnew file mode 100644index 000000000..79378a9bc--- /dev/null+++ b/apps/api/src/lib/hosted-bash/docs.ts@@ -0,0 +1,108 @@+import type { InitialFiles } from "just-bash";+import { logger } from "../logger";++const DOCS_ORIGIN = "https://docs.firecrawl.dev";+const INDEX_URL = `${DOCS_ORIGIN}/llms.txt`;+const TTL_MS = 60 * 60 * 1000;+const RETRY_MS = 60 * 1000;+const FETCH_TIMEOUT_MS = 10_000;+const LINK = /\]\((https:\/\/docs\.firecrawl\.dev\/[^)\s]+)\)/g;++/** Docs text by URL, shared by every shell on this server for an hour. */+const cache = new Map<string, { at: number; text: Promise<string> }>();++function fetchText(url: string): Promise<string> {+ const hit = cache.get(url);+ if (hit && Date.now() - hit.at < TTL_MS) return hit.text;+ const text = fetch(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })+ .then(res => {+ if (!res.ok) throw new Error(`${url}: HTTP ${res.status}`);+ return res.text();+ })+ .catch(error => {+ // A failure is not cached, so the next read tries again.+ if (cache.get(url)?.text === text) cache.delete(url);+ throw error;+ });+ cache.set(url, { at: Date.now(), text });+ return text;+}++/**+ * Every English page path, found by walking llms.txt and its English+ * sub-indexes. An index that fails to load is skipped and reported, so the+ * pages found elsewhere are kept.+ */+async function walkIndexes(): Promise<{ pages: string[]; complete: boolean }> {+ const seen = new Set<string>();+ const pages = new Set<string>();+ let complete = true;+ const walk = async (url: string): Promise<void> => {+ if (seen.has(url)) return;+ seen.add(url);+ let index: string;+ try {+ index = await fetchText(url);+ } catch (error) {+ complete = false;+ logger.warn("Hosted bash could not load a docs index", {+ module: "hosted-bash",+ url,+ error,+ });+ return;+ }+ const links = [...index.matchAll(LINK)].map(m => m[1]);+ for (const link of links) {+ if (link.endsWith(".md")) pages.add(new URL(link).pathname);+ }+ await Promise.all(+ links+ .filter(+ link =>+ link.endsWith(".txt") &&+ new URL(link).pathname.startsWith("/_llms/en"),+ )+ .map(walk),+ );+ };+ await walk(INDEX_URL);+ return { pages: [...pages].sort(), complete };+}++let known: string[] | null = null;+let refreshAt = 0;+let refresh: Promise<string[]> | null = null;++/**+ * The page list, refreshed hourly in the background. A command only waits for+ * it on a server that has never listed the docs; after that a slow or failing+ * docs site leaves the last list in place and is retried a minute later.+ */+function listPages(): Promise<string[]> {+ if (!refresh && Date.now() >= refreshAt) {+ refresh = walkIndexes()+ .then(({ pages, complete }) => {+ refreshAt = Date.now() + (complete ? TTL_MS : RETRY_MS);+ // A partial walk never replaces a fuller list.+ if (complete || pages.length >= (known?.length ?? 0)) known = pages;+ return known ?? pages;+ })+ .finally(() => {+ refresh = null;+ });+ }+ return known ? Promise.resolve(known) : (refresh as Promise<string[]>);+}++/**+ * The live docs.firecrawl.dev pages as files under /docs. Only the page list is+ * needed to start a shell; a page is fetched when a command first reads it.+ */+export async function docsFiles(): Promise<InitialFiles> {+ const files: InitialFiles = { "/docs/llms.txt": () => fetchText(INDEX_URL) };+ for (const path of await listPages().catch(() => [])) {+ files[`/docs${path}`] = () => fetchText(`${DOCS_ORIGIN}${path}`);+ }+ return files;+}diff --git a/apps/api/src/lib/hosted-bash/firecrawl-command.ts b/apps/api/src/lib/hosted-bash/firecrawl-command.tsnew file mode 100644index 000000000..15155ac83--- /dev/null+++ b/apps/api/src/lib/hosted-bash/firecrawl-command.ts@@ -0,0 +1,657 @@+import { defineCommand, type IFileSystem } from "just-bash";++const ORIGIN = "bash";+const INTEGRATION = "_firecrawl-bash";+const INSTALL = "npx -y firecrawl-cli@latest init";+const SEARCH_SOURCES = ["web", "images", "news"];+const SEARCH_CATEGORIES = ["research", "pdf", "developer", "gov"];+// Every format the CLI accepts; the hosted shell serves SCRAPE_FORMATS of them.+const CLI_FORMATS = [+ "markdown",+ "html",+ "rawHtml",+ "links",+ "images",+ "screenshot",+ "summary",+ "changeTracking",+ "json",+ "attributes",+ "branding",+];+const SCRAPE_FORMATS = [+ "markdown",+ "html",+ "rawHtml",+ "links",+ "images",+ "summary",+];+const API_KEY_FLAG =+ "--api-key isn't accepted in the hosted shell. Send your key in the HTTP request's Authorization: Bearer fc-... header instead";+const KEY_NEEDED =+ "Firecrawl's free tier without an API key isn't available from your network (it needs an IPv4 address). Create a free API key at https://www.firecrawl.dev/signin and send it as Authorization: Bearer fc-...";++export interface FirecrawlCaller {+ /** This server's own address: calls go through the same routes a REST caller uses. */+ baseUrl: string;+ ip: string | null;+ /** The caller's Authorization header as sent, so the API parses it as it would directly. */+ authorization: string | null;+ /** Caller headers kept on each call: User-Agent and a World ID credential. */+ headers: Record<string, string>;+ maxCalls: number;+ /** When the command must be done; each call asks the API to finish before it. */+ deadline: number;+}++// Room to read the response and print it before the command's deadline.+const CALL_MARGIN_MS = 3_000;++type Endpoint = "search" | "scrape";++/** Calls the API as the shell's caller and records what was called, for analytics. */+export class FirecrawlClient {+ readonly endpoints: Endpoint[] = [];+ readonly failedEndpoints: Endpoint[] = [];++ constructor(private readonly caller: FirecrawlCaller) {}++ async post(+ endpoint: Endpoint,+ body: Record<string, unknown>,+ signal?: AbortSignal,+ ): Promise<any> {+ const { baseUrl, ip, authorization, headers, maxCalls, deadline } =+ this.caller;+ if (this.endpoints.length >= maxCalls) {+ throw new Error(+ `call limit reached (${maxCalls} Firecrawl calls per command)`,+ );+ }+ // A call the API is still working on at the deadline would be billed but lost.+ const timeout = deadline - Date.now() - CALL_MARGIN_MS;+ if (timeout < 1_000) {+ throw new Error("not enough time left before the command's deadline");+ }+ this.endpoints.push(endpoint);+ const res = await fetch(`${baseUrl}/v2/${endpoint}`, {+ method: "POST",+ headers: {+ ...headers,+ "Content-Type": "application/json",+ ...(authorization ? { Authorization: authorization } : {}),+ // This request arrives from loopback, a trusted hop under+ // EXPRESS_TRUST_PROXY, so auth and quotas see the caller's IP.+ ...(ip ? { "X-Forwarded-For": ip } : {}),+ },+ body: JSON.stringify({+ ...body,+ timeout,+ origin: ORIGIN,+ integration: INTEGRATION,+ }),+ signal,+ });+ const json: any = await res.json().catch(() => ({}));+ if (!res.ok || json?.success === false) {+ this.failedEndpoints.push(endpoint);+ // Keyless refuses an IPv6 caller with a bare 401.+ const bare = !json?.error || json.error === "Unauthorized";+ if (!authorization && res.status === 401 && bare) {+ throw new Error(KEY_NEEDED);+ }+ throw new Error(apiError(json, res.status));+ }+ return json;+ }+}++/** The API's error, with the first few validation issues it lists. */+function apiError(json: any, status: number): string {+ const message = json?.error || `Firecrawl request failed (HTTP ${status})`;+ const issues: string[] = Array.isArray(json?.details)+ ? json.details+ .slice(0, 3)+ .filter((d: any) => typeof d?.message === "string")+ .map((d: any) =>+ Array.isArray(d.path) && d.path.length+ ? `${d.path.join(".")}: ${d.message}`+ : d.message,+ )+ : [];+ return issues.length ? `${message} (${issues.join("; ")})` : message;+}++export const FIRECRAWL_HELP = `Usage: firecrawl [command] [options]++A hosted subset of the Firecrawl CLI. The flags and output follow the real CLI+(/docs/sdks/cli.md); install it for everything else: ${INSTALL}++Commands:+ search <query> Search the web+ --limit <number> Maximum number of results (default: 5)+ --sources <sources> Comma-separated sources: web, images, news (default: web)+ --categories <categories> Comma-separated categories: ${SEARCH_CATEGORIES.join(", ")}+ --tbs <value> Time-based search: qdr:h, qdr:d, qdr:w, qdr:m, qdr:y+ --location <location> Location for geo-targeting+ --country <code> ISO country code for geo-targeting+ --scrape Enable scraping of search results+ --scrape-formats <formats> Comma-separated scrape formats when --scrape is enabled (default: markdown)+ -o, --output <path> Output file path (default: stdout)+ --json Output as compact JSON+ --pretty Output as pretty JSON++ scrape [urls...] Scrape URLs. Multiple URLs are saved to .firecrawl/+ -f, --format <formats> Output format(s): ${SCRAPE_FORMATS.join(", ")}.+ Single format outputs raw content; multiple formats output JSON.+ -H, --html Output raw HTML (shortcut for --format html)+ -S, --summary Output summary (shortcut for --format summary)+ --only-main-content Include only the main content (the API default)+ --full-page Keep the whole page, including navigation, headers and footers+ --wait-for <ms> Wait time before scraping in milliseconds+ --include-tags <tags> Comma-separated list of tags to include+ --exclude-tags <tags> Comma-separated list of tags to exclude+ --max-age <milliseconds> Maximum age of cached content in milliseconds+ --country <code> ISO country code for geo-targeted scraping+ -o, --output <path> Output file path (default: stdout)+ --json Output as JSON format+ --pretty Pretty print JSON output++ help Display this help+`;++/** Thrown for anything the hosted subset leaves out; exits 2, unlike a failed call. */+class Unsupported extends Error {}++type Flags = Record<string, string | true>;++interface FlagSpec {+ values: string[];+ booleans: string[];+ aliases: Record<string, string>;+}++const SEARCH_FLAGS: FlagSpec = {+ values: [+ "limit",+ "sources",+ "categories",+ "tbs",+ "location",+ "country",+ "scrape-formats",+ "output",+ ],+ booleans: ["scrape", "json", "pretty"],+ aliases: { o: "output" },+};++const SCRAPE_FLAGS: FlagSpec = {+ values: [+ "url",+ "format",+ "wait-for",+ "include-tags",+ "exclude-tags",+ "max-age",+ "country",+ "output",+ ],+ booleans: [+ "html",+ "summary",+ "only-main-content",+ "full-page",+ "json",+ "pretty",+ ],+ aliases: {+ u: "url",+ f: "format",+ formats: "format",+ H: "html",+ S: "summary",+ o: "output",+ },+};++function parseArgs(args: string[], spec: FlagSpec) {+ const positional: string[] = [];+ const flags: Flags = {};+ for (let i = 0; i < args.length; i++) {+ const arg = args[i];+ if (arg === "--") {+ positional.push(...args.slice(i + 1));+ break;+ }+ const match = /^(?:--([\w-]+)|-([A-Za-z]))(?:=(.*))?$/s.exec(arg);+ if (!match) {+ positional.push(arg);+ continue;+ }+ const raw = match[1] ?? match[2];+ const name = spec.aliases[raw] ?? raw;+ const shown = match[1] ? `--${raw}` : `-${raw}`;+ if (spec.booleans.includes(name)) {+ flags[name] = true;+ } else if (!spec.values.includes(name)) {+ throw new Unsupported(`option '${shown}' isn't in the hosted shell`);+ } else if (match[3] !== undefined) {+ flags[name] = match[3];+ } else if (i + 1 < args.length) {+ flags[name] = args[++i];+ } else {+ throw new Error(`option '${shown}' argument missing`);+ }+ }+ return { positional, flags };+}++const text = (flags: Flags, name: string) =>+ typeof flags[name] === "string" ? (flags[name] as string) : undefined;++function integer(flags: Flags, name: string): number | undefined {+ const value = text(flags, name);+ if (value === undefined) return undefined;+ const parsed = Number.parseInt(value, 10);+ if (Number.isNaN(parsed)) {+ throw new Error(`option '--${name}' expects a number`);+ }+ return parsed;+}++const list = (value: string | undefined) =>+ value+ ?.split(",")+ .map(item => item.trim())+ .filter(Boolean);++interface Context {+ client: FirecrawlClient;+ fs: IFileSystem;+ cwd: string;+ signal?: AbortSignal;+ stderr: string[];+}++/** The CLI's writeOutput: a file when a path is given, otherwise stdout ending in a newline. */+async function output(c: Context, content: string, path: string | undefined) {+ if (!path) return content.endsWith("\n") ? content : `${content}\n`;+ const absolute = path.startsWith("/")+ ? path+ : `${c.cwd.replace(/\/$/, "")}/${path}`;+ const dir = absolute.slice(0, absolute.lastIndexOf("/")) || "/";+ await c.fs.mkdir(dir, { recursive: true });+ await c.fs.writeFile(absolute, content);+ return "";+}++const wantsJson = (flags: Flags, path: string | undefined) =>+ Boolean(flags.json) || Boolean(path?.toLowerCase().endsWith(".json"));++const stringify = (flags: Flags, value: unknown) =>+ flags.pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value);++function scrapedContent(markdown: unknown): string[] {+ if (typeof markdown !== "string" || !markdown) return [];+ const indented = markdown+ .split("\n")+ .map(line => ` ${line}`)+ .join("\n");+ return ["", " --- Content ---", indented, " --- End Content ---"];+}++function readableSearch(data: any, labelWeb: boolean): string {+ const lines: string[] = [];+ if (data.web?.length) {+ if (labelWeb) lines.push("=== Web Results ===", "");+ for (const r of data.web) {+ lines.push(r.title || "Untitled", ` URL: ${r.url}`);+ if (r.description) lines.push(` ${r.description}`);+ if (r.category) lines.push(` Category: ${r.category}`);+ lines.push(...scrapedContent(r.markdown), "");+ }+ }+ if (data.images?.length) {+ if (lines.length) lines.push("");+ lines.push("=== Image Results ===", "");+ for (const r of data.images) {+ lines.push(+ r.title || "Untitled",+ ` Image URL: ${r.imageUrl}`,+ ` Source: ${r.url}`,+ );+ if (r.imageWidth && r.imageHeight) {+ lines.push(` Size: ${r.imageWidth}x${r.imageHeight}`);+ }+ lines.push("");+ }+ }+ if (data.news?.length) {+ if (lines.length) lines.push("");+ lines.push("=== News Results ===", "");+ for (const r of data.news) {+ lines.push(r.title || "Untitled", ` URL: ${r.url}`);+ if (r.date) lines.push(` Date: ${r.date}`);+ if (r.snippet) lines.push(` ${r.snippet}`);+ lines.push(...scrapedContent(r.markdown), "");+ }+ }+ return lines.join("\n");+}++async function search(c: Context, args: string[]): Promise<string> {+ const { positional, flags } = parseArgs(args, SEARCH_FLAGS);+ const query = positional.join(" ");+ if (!query) throw new Error("missing required argument 'query'");+ const named = list(text(flags, "sources"))?.map(s => s.toLowerCase());+ const sources = named?.length ? [...new Set(named)] : ["web"];+ for (const source of sources) {+ if (!SEARCH_SOURCES.includes(source)) {+ throw new Unsupported(+ `source "${source}" isn't in the hosted shell, which has ${SEARCH_SOURCES.join(", ")}`,+ );+ }+ }+ const categories = list(text(flags, "categories"))?.map(c => c.toLowerCase());+ for (const category of categories ?? []) {+ if (category === "github") {+ throw new Error(+ "The github category was removed. Use --categories developer",+ );+ }+ if (!SEARCH_CATEGORIES.includes(category)) {+ throw new Error(+ `Invalid category "${category}". Valid categories are: ${SEARCH_CATEGORIES.join(", ")}`,+ );+ }+ }+ const scrapeFormats = list(text(flags, "scrape-formats")) ?? ["markdown"];+ const optional = (key: string, flag = key) =>+ text(flags, flag) ? { [key]: text(flags, flag) } : {};++ const envelope = await c.client.post(+ "search",+ {+ query,+ limit: integer(flags, "limit") ?? 5,+ sources: sources.map(type => ({ type })),+ // The CLI gets highlighted descriptions by default; ask for the same.+ highlights: true,+ ...(categories ? { categories: categories.map(type => ({ type })) } : {}),+ ...optional("tbs"),+ ...optional("location"),+ ...optional("country"),+ ...(flags.scrape+ ? { scrapeOptions: { formats: scrapeFormats.map(type => ({ type })) } }+ : {}),+ },+ c.signal,+ );+ const data = envelope.data ?? {};+ const path = text(flags, "output");+ if (!data.web?.length && !data.images?.length && !data.news?.length) {+ return "No results found.\n";+ }+ if (flags.json || flags.pretty) {+ const result = {+ success: true,+ data,+ ...(envelope.warning ? { warning: envelope.warning } : {}),+ ...(envelope.id ? { id: envelope.id } : {}),+ ...(envelope.creditsUsed !== undefined+ ? { creditsUsed: envelope.creditsUsed }+ : {}),+ };+ return output(c, stringify(flags, result), path);+ }+ return output(c, readableSearch(data, sources.length > 1), path);+}++function scrapeBody(url: string, formats: string[], flags: Flags) {+ const waitFor = integer(flags, "wait-for");+ const maxAge = integer(flags, "max-age");+ const includeTags = list(text(flags, "include-tags"));+ const excludeTags = list(text(flags, "exclude-tags"));+ const country = text(flags, "country");+ return {+ url,+ formats,+ ...(flags["full-page"]+ ? { onlyMainContent: false }+ : flags["only-main-content"]+ ? { onlyMainContent: true }+ : {}),+ ...(waitFor !== undefined ? { waitFor } : {}),+ ...(includeTags ? { includeTags } : {}),+ ...(excludeTags ? { excludeTags } : {}),+ ...(maxAge !== undefined ? { maxAge } : {}),+ ...(country ? { location: { country } } : {}),+ };+}++/** A single text format prints as raw content, as the CLI's extractContent does. */+function rawContent(data: any, format: string): string | null {+ const value =+ format === "html" || format === "rawHtml"+ ? data.html || data.rawHtml+ : data[format];+ if (Array.isArray(value)) return value.join("\n");+ return typeof value === "string" && value ? value : null;+}++/** The CLI's parseFormats: comma-separated, any case, in the API's casing. */+function parseFormats(value: string): string[] {+ const formats = value+ .split(",")+ .map(format => format.trim())+ .filter(Boolean)+ .map(format => {+ const known = CLI_FORMATS.find(+ name => name.toLowerCase() === format.toLowerCase(),+ );+ if (!known) {+ throw new Error(+ `Invalid format(s): ${format}. Valid formats are: ${CLI_FORMATS.join(", ")}`,+ );+ }+ return known;+ });+ return [...new Set(formats)];+}++function isFormatList(value: string): boolean {+ try {+ return parseFormats(value).length > 0;+ } catch {+ return false;+ }+}++const normalizeUrl = (url: string) =>+ /^https?:\/\//i.test(url) ? url : `https://${url}`;++/** The CLI's isScrapeUrl: an http(s) URL, or a host with a dot, localhost or an IPv6 literal. */+function isScrapeUrl(value: string): boolean {+ if (!/^https?:\/\//i.test(value)) {+ const host = value.split(/[/?#]/, 1)[0];+ const hostLike =+ host.includes(".") ||+ /^localhost(?::\d+)?$/i.test(host) ||+ host.startsWith("[");+ if (!hostLike) return false;+ }+ try {+ return Boolean(new URL(normalizeUrl(value)).hostname);+ } catch {+ return false;+ }+}++/** The CLI's isUrl, which decides when `firecrawl <url>` means scrape. */+function isShorthandUrl(value: string): boolean {+ if (/^https?:\/\//i.test(value)) return true;+ return (+ value.includes(".") &&+ !value.startsWith("-") &&+ /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\.[a-zA-Z]{2,}(\/.*)?$/.test(+ value,+ )+ );+}++function urlToFilename(url: string): string {+ try {+ const parsed = new URL(url);+ const host = parsed.hostname.replace(/^www\./, "");+ const path = parsed.pathname.replace(/^\/|\/$/g, "").replace(/\//g, "-");+ return path ? `${host}-${path}.md` : `${host}.md`;+ } catch {+ return `${url.replace(/[^a-zA-Z0-9.-]/g, "_")}.md`;+ }+}++async function scrape(c: Context, args: string[]): Promise<string> {+ const { positional, flags } = parseArgs(args, SCRAPE_FLAGS);+ // As in the CLI, each word is a URL or formats: `firecrawl scrape example.com markdown,links`.+ const urls: string[] = [];+ const positionalFormats: string[] = [];+ for (const word of [+ ...positional,+ ...(text(flags, "url") ? [text(flags, "url") as string] : []),+ ]) {+ if (isScrapeUrl(word)) {+ const url = normalizeUrl(word);+ if (!urls.includes(url)) urls.push(url);+ } else if (isFormatList(word))+ positionalFormats.push(...parseFormats(word));+ else {+ throw new Error(+ `${JSON.stringify(word)} isn't a URL. For a website, use an http(s) URL, for example https://example.com`,+ );+ }+ }+ if (!urls.length) {+ throw new Error(+ "Provide a URL to scrape, e.g. firecrawl scrape https://example.com",+ );+ }+ if (flags["only-main-content"] && flags["full-page"]) {+ throw new Error("Use either --only-main-content or --full-page, not both");+ }+ const formats = positionalFormats.length+ ? [...new Set(positionalFormats)]+ : flags.html+ ? ["html"]+ : flags.summary+ ? ["summary"]+ : text(flags, "format")+ ? parseFormats(text(flags, "format") as string)+ : ["markdown"];+ for (const format of formats) {+ if (!SCRAPE_FORMATS.includes(format)) {+ throw new Unsupported(+ `format "${format}" isn't in the hosted shell, which has ${SCRAPE_FORMATS.join(", ")}`,+ );+ }+ }++ if (urls.length > 1) {+ c.stderr.push(`Scraping ${urls.length} URLs...`);+ let done = 0;+ let failed = 0;+ await Promise.all(+ urls.map(async url => {+ try {+ const body = scrapeBody(url, formats, flags);+ const data = (await c.client.post("scrape", body, c.signal)).data;+ const path = `.firecrawl/${urlToFilename(url)}`;+ // As the CLI does: markdown, or the whole response, whatever the format.+ await output(c, data?.markdown || JSON.stringify(data ?? {}), path);+ c.stderr.push(`[${++done}/${urls.length}] Saved: ${path}`);+ } catch (error) {+ failed++;+ c.stderr.push(+ `[${++done}/${urls.length}] Error: ${url} - ${(error as Error).message}`,+ );+ }+ }),+ );+ c.stderr.push(+ "",+ `Completed: ${urls.length - failed}/${urls.length} succeeded${failed ? `, ${failed} failed` : ""}`,+ );+ if (failed === urls.length) throw new Error("every URL failed");+ return "";+ }++ const body = scrapeBody(urls[0], formats, flags);+ const data = (await c.client.post("scrape", body, c.signal)).data ?? {};+ const path = text(flags, "output");+ if (wantsJson(flags, path)) return output(c, stringify(flags, data), path);+ const raw = formats.length === 1 ? rawContent(data, formats[0]) : null;+ if (raw !== null) return output(c, raw, path);+ const picked = Object.fromEntries(+ [...formats, "metadata"]+ .filter(key => data[key] !== undefined)+ .map(key => [key, data[key]]),+ );+ return output(+ c,+ stringify(flags, formats.length === 1 ? data : picked),+ path,+ );+}++export function firecrawlCommand(client: FirecrawlClient) {+ return defineCommand("firecrawl", async (argv, ctx) => {+ // As in the CLI, `firecrawl <url>` is short for `firecrawl scrape <url>`.+ const [name, ...rest] = isShorthandUrl(argv[0] ?? "")+ ? ["scrape", ...argv]+ : argv;+ const stderr: string[] = [];+ const result = (stdout: string, exitCode: number) => ({+ stdout,+ stdoutKind: "text" as const,+ stderr: stderr.length ? `${stderr.join("\n")}\n` : "",+ exitCode,+ });+ // A bare `help` is only the help command up front: `search help desk` is a query.+ const end = rest.indexOf("--");+ const options = end === -1 ? rest : rest.slice(0, end);+ if ([name, ...options].some(arg => /^(--api-key|-k)(=|$)/.test(arg))) {+ stderr.push(`Error: ${API_KEY_FLAG}`);+ return result("", 1);+ }+ const asksHelp = options.some(arg => arg === "--help" || arg === "-h");+ if (!name || ["help", "--help", "-h"].includes(name) || asksHelp) {+ return result(FIRECRAWL_HELP, 0);+ }+ const hint = `Install the Firecrawl CLI for the rest: ${INSTALL} (see /docs/sdks/cli.md)`;+ if (name !== "search" && name !== "scrape") {+ stderr.push(+ `firecrawl ${name} isn't in this hosted shell, which has search and scrape. ${hint}`,+ );+ return result("", 2);+ }+ const c: Context = {+ client,+ fs: ctx.fs,+ cwd: ctx.cwd,+ signal: ctx.signal,+ stderr,+ };+ try {+ return result(await (name === "search" ? search : scrape)(c, rest), 0);+ } catch (error) {+ const message = error instanceof Error ? error.message : String(error);+ if (error instanceof Unsupported) {+ stderr.push(`Error: ${message}. ${hint}`);+ return result("", 2);+ }+ stderr.push(`Error: ${message}`);+ return result("", 1);+ }+ });+}diff --git a/apps/api/src/lib/hosted-bash/limits.ts b/apps/api/src/lib/hosted-bash/limits.tsnew file mode 100644index 000000000..759eb9acc--- /dev/null+++ b/apps/api/src/lib/hosted-bash/limits.ts@@ -0,0 +1,3 @@+export const TIMEOUT_MS = 30_000;+export const MAX_OUTPUT_BYTES = 256_000;+export const MAX_FIRECRAWL_CALLS = 25;diff --git a/apps/api/src/lib/hosted-bash/shell-worker.ts b/apps/api/src/lib/hosted-bash/shell-worker.tsnew file mode 100644index 000000000..934058073--- /dev/null+++ b/apps/api/src/lib/hosted-bash/shell-worker.ts@@ -0,0 +1,188 @@+import { Bash, type CommandName, type InitialFiles } from "just-bash";+import {+ FIRECRAWL_HELP,+ FirecrawlClient,+ firecrawlCommand,+ type FirecrawlCaller,+} from "./firecrawl-command";+import { MAX_OUTPUT_BYTES } from "./limits";++/** What the API process hands the shell process: a command, its caller and the docs pages to mount. */+export interface ShellJob {+ command: string;+ caller: FirecrawlCaller;+ docs: string[];+ deadline: number;+}++export interface SandboxResult {+ stdout: string;+ stderr: string;+ exitCode: number;+ truncated: boolean;+ /** Firecrawl endpoints the command called, and the ones that failed. */+ endpoints: string[];+ failedEndpoints: string[];+}++export type WorkerMessage =+ | { type: "doc"; id: number; path: string }+ | { type: "done"; result: SandboxResult };++export type ParentMessage =+ | { type: "job"; job: ShellJob }+ | { type: "doc"; id: number; text?: string; error?: string };++const MIB = 1024 * 1024;++// Text tools only: no archives, databases, Python or JavaScript. bash, sh, awk+// and xargs still run commands, under the same execution limits.+const COMMANDS: CommandName[] = [+ "echo",+ "cat",+ "printf",+ "ls",+ "mkdir",+ "touch",+ "rm",+ "cp",+ "mv",+ "pwd",+ "head",+ "tail",+ "wc",+ "stat",+ "grep",+ "fgrep",+ "egrep",+ "rg",+ "sed",+ "awk",+ "sort",+ "uniq",+ "comm",+ "cut",+ "paste",+ "tr",+ "rev",+ "nl",+ "fold",+ "column",+ "join",+ "tee",+ "find",+ "basename",+ "dirname",+ "tree",+ "du",+ "env",+ "xargs",+ "true",+ "false",+ "bash",+ "sh",+ "jq",+ "base64",+ "date",+ "sleep",+ "seq",+ "expr",+ "tac",+ "which",+ "help",+];++const README = `# Firecrawl bash++A stateless virtual shell with a hosted subset of the Firecrawl CLI as \`firecrawl\`.+Each request starts from this fresh filesystem; nothing persists between requests.++ /README.md this file+ /docs/ the live Firecrawl docs (docs.firecrawl.dev) as markdown, fetched when read.+ Start with /docs/llms.txt, or grep -l -r <term> /docs/features /docs/sdks++${FIRECRAWL_HELP}`;++function truncate(text: string): [string, boolean] {+ const bytes = Buffer.from(text, "utf8");+ if (bytes.length <= MAX_OUTPUT_BYTES) return [text, false];+ const kept = bytes.subarray(0, MAX_OUTPUT_BYTES).toString("utf8");+ return [`${kept}\n[truncated at ${MAX_OUTPUT_BYTES} bytes]\n`, true];+}++const send = (message: WorkerMessage) => process.send?.(message);++// Docs pages are read through the API process, which caches them for every shell.+const pending = new Map<+ number,+ { resolve: (text: string) => void; reject: (error: Error) => void }+>();+let nextId = 0;+process.on("message", (message: ParentMessage) => {+ if (message.type === "job") {+ void run(message.job)+ .catch(+ (error: unknown): SandboxResult => ({+ stdout: "",+ stderr: `bash: ${error instanceof Error ? error.message : String(error)}\n`,+ exitCode: 1,+ truncated: false,+ endpoints: [],+ failedEndpoints: [],+ }),+ )+ .then(result => send({ type: "done", result }));+ return;+ }+ const waiter = pending.get(message.id);+ if (!waiter) return;+ pending.delete(message.id);+ if (message.error === undefined) waiter.resolve(message.text ?? "");+ else waiter.reject(new Error(message.error));+});+const readDoc = (path: string) =>+ new Promise<string>((resolve, reject) => {+ const id = nextId++;+ pending.set(id, { resolve, reject });+ send({ type: "doc", id, path });+ });++async function run(job: ShellJob): Promise<SandboxResult> {+ const client = new FirecrawlClient(job.caller);+ const files: InitialFiles = { "/README.md": README };+ for (const path of job.docs) files[path] = () => readDoc(path);+ const bash = new Bash({+ files,+ cwd: "/",+ env: { HOME: "/", PWD: "/" },+ commands: COMMANDS,+ customCommands: [firecrawlCommand(client)],+ executionLimitProfile: "hardened",+ // The API process also kills this process at the deadline, since a single+ // long command never reaches just-bash's own check.+ executionLimits: {+ maxExecutionTimeMs: Math.max(1, job.deadline - Date.now()),+ maxFileSystemBytes: 32 * MIB,+ maxStringLength: MIB,+ maxHeredocSize: MIB,+ maxOutputSize: 2 * MIB,+ maxLiveBytes: 64 * MIB,+ maxInputBytes: 32 * MIB,+ },+ });+ const outcome = await bash.exec(job.command).catch((error: unknown) => ({+ stdout: "",+ stderr: `${error instanceof Error ? error.message : String(error)}\n`,+ exitCode: 1,+ }));+ const [stdout, stdoutCut] = truncate(outcome.stdout);+ const [stderr, stderrCut] = truncate(outcome.stderr);+ return {+ stdout,+ stderr,+ exitCode: outcome.exitCode,+ truncated: stdoutCut || stderrCut,+ endpoints: client.endpoints,+ failedEndpoints: client.failedEndpoints,+ };+}diff --git a/apps/api/src/lib/hosted-bash/shell.ts b/apps/api/src/lib/hosted-bash/shell.tsnew file mode 100644index 000000000..6b5c48f99--- /dev/null+++ b/apps/api/src/lib/hosted-bash/shell.ts@@ -0,0 +1,155 @@+import type { InitialFiles } from "just-bash";+import { fork } from "node:child_process";+import { readFile } from "node:fs/promises";+import { extname, join } from "node:path";+import { docsFiles } from "./docs";+import type { FirecrawlCaller } from "./firecrawl-command";+import { MAX_FIRECRAWL_CALLS, TIMEOUT_MS } from "./limits";+import type {+ ParentMessage,+ SandboxResult,+ ShellJob,+ WorkerMessage,+} from "./shell-worker";++// Compiled, this is shell-worker.js; under tsx or vitest it is the .ts source.+const WORKER_FILE = join(__dirname, `shell-worker${extname(__filename)}`);+const MAX_SHELLS = 4;+const HEAP_MB = 192;+// Under tsx the transpiler alone holds about 200 MB.+const MAX_RSS_KB = (WORKER_FILE.endsWith(".ts") ? 576 : 320) * 1024;+const RSS_POLL_MS = 50;+const KILL_GRACE_MS = 2_000;+const DOCS_WAIT_MS = 5_000;++let running = 0;++/** Every shell slot on this server is taken. */+export class ShellBusyError extends Error {}++export interface ShellResult extends SandboxResult {+ durationMs: number;+}++type Caller = Omit<FirecrawlCaller, "maxCalls" | "deadline">;++/**+ * Runs one command in a fresh shell in its own process, so a command that+ * spins or runs out of memory only costs that process, killed at the deadline.+ * A worker thread is not enough: one huge allocation aborts the whole process.+ */+export async function runShell(+ command: string,+ caller: Caller,+ signal: AbortSignal,+): Promise<ShellResult> {+ if (running >= MAX_SHELLS) throw new ShellBusyError();+ running++;+ try {+ const started = Date.now();+ const deadline = started + TIMEOUT_MS;+ const docs = await Promise.race([+ docsFiles(),+ new Promise<InitialFiles>(resolve =>+ setTimeout(() => resolve({}), DOCS_WAIT_MS).unref(),+ ),+ new Promise<InitialFiles>(resolve => {+ if (signal.aborted) resolve({});+ signal.addEventListener("abort", () => resolve({}), { once: true });+ }),+ ]);+ if (signal.aborted) {+ return {+ stdout: "",+ stderr: "bash: the request was closed\n",+ exitCode: 130,+ truncated: false,+ endpoints: [],+ failedEndpoints: [],+ durationMs: Date.now() - started,+ };+ }+ const job: ShellJob = {+ command,+ caller: { ...caller, maxCalls: MAX_FIRECRAWL_CALLS, deadline },+ docs: Object.keys(docs),+ deadline,+ };+ return await new Promise<ShellResult>(resolve => {+ const child = fork(WORKER_FILE, [], {+ env: {},+ execArgv: [+ `--max-old-space-size=${HEAP_MB}`,+ ...(WORKER_FILE.endsWith(".ts") ? ["--require", "tsx/cjs"] : []),+ ],+ stdio: ["ignore", "ignore", "ignore", "ipc"],+ });+ let settled = false;+ const send = (message: ParentMessage) => {+ if (!settled && child.connected) child.send(message);+ };+ const finish = (result: SandboxResult) => {+ if (settled) return;+ settled = true;+ clearTimeout(timer);+ clearInterval(watch);+ signal.removeEventListener("abort", onAbort);+ child.kill("SIGKILL");+ resolve({ ...result, durationMs: Date.now() - started });+ };+ const fail = (exitCode: number, message: string) =>+ finish({+ stdout: "",+ stderr: `bash: ${message}\n`,+ exitCode,+ truncated: false,+ endpoints: [],+ failedEndpoints: [],+ });+ const timer = setTimeout(+ () =>+ fail(+ 124,+ `execution exceeded the ${TIMEOUT_MS / 1000} second deadline`,+ ),+ deadline + KILL_GRACE_MS - Date.now(),+ );+ const onAbort = () => fail(130, "the request was closed");+ signal.addEventListener("abort", onAbort);+ // The heap flag misses large strings, so resident memory is watched too.+ // Linux only; elsewhere the read fails and the heap flag is the only cap.+ const watch = setInterval(() => {+ readFile(`/proc/${child.pid}/status`, "utf8").then(+ status => {+ const rss = Number(/VmRSS:\s+(\d+)/.exec(status)?.[1] ?? 0);+ if (rss > MAX_RSS_KB) fail(1, "memory limit exceeded");+ },+ () => {},+ );+ }, RSS_POLL_MS);++ child.on("message", (message: WorkerMessage) => {+ if (message.type === "done") return finish(message.result);+ const read = docs[message.path] as (() => Promise<string>) | undefined;+ (read ? read() : Promise.reject(new Error("No such file"))).then(+ text => send({ type: "doc", id: message.id, text }),+ (error: Error) =>+ send({ type: "doc", id: message.id, error: error.message }),+ );+ });+ child.on("error", error => fail(1, error.message));+ // V8 aborts the process when the command outgrows its heap.+ child.on("exit", (code, killSignal) =>+ fail(+ 1,+ killSignal === "SIGABRT" || code === 134+ ? "memory limit exceeded"+ : "the shell stopped unexpectedly",+ ),+ );+ send({ type: "job", job });+ });+ } finally {+ running--;+ }+}