Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(log_job): start saving jobs to GCS (#1424)
Gergő Móriczbd1c1b0012e8
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 3feec21f8..357914831 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -48,7 +48,7 @@"supertest": "^6.3.3","ts-jest": "^29.1.1","ts-node": "^10.9.1",- "typescript": "^5.4.2"+ "typescript": "^5.8.3"},"dependencies": {"@ai-sdk/openai": "^1.1.13",@@ -59,6 +59,7 @@"@bull-board/express": "^5.20.5","@devil7softwares/pos": "^1.0.2","@dqbd/tiktoken": "^1.0.17",+ "@google-cloud/storage": "^7.16.0","@nangohq/node": "^0.40.8","@pinecone-database/pinecone": "^4.0.0","@sentry/cli": "^2.33.1",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex fdd8aa142..508fb7cd0 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -32,6 +32,9 @@ importers:'@dqbd/tiktoken':specifier: ^1.0.17version: 1.0.17+ '@google-cloud/storage':+ specifier: ^7.16.0+ version: 7.16.0([email protected])'@nangohq/node':specifier: ^0.40.8version: 0.40.8@@ -196,7 +199,7 @@ importers:version: 0.0.10puppeteer:specifier: ^22.12.1- version: 22.12.1([email protected])+ version: 22.12.1([email protected])rate-limiter-flexible:specifier: 2.4.2version: 2.4.2@@ -296,7 +299,7 @@ importers:version: 6.0.2jest:specifier: ^29.6.3- version: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ version: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))jest-fetch-mock:specifier: ^3.0.3version: 3.0.3([email protected])@@ -311,13 +314,13 @@ importers:version: 6.3.4ts-jest:specifier: ^29.1.1- version: 29.1.4(@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected])+ version: 29.1.4(@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected])ts-node:specifier: ^10.9.1- version: 10.9.2(@types/[email protected])([email protected])+ version: 10.9.2(@types/[email protected])([email protected])typescript:- specifier: ^5.4.2- version: 5.4.5+ specifier: ^5.8.3+ version: 5.8.3packages:@@ -767,6 +770,22 @@ packages:'@dqbd/[email protected]':resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}+ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}+ engines: {node: '>=14.0.0'}++ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==}+ engines: {node: '>=14.0.0'}++ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==}+ engines: {node: '>=14'}++ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-7/5LRgykyOfQENcm6hDKP8SX/u9XxE5YOiWOkgkwcoO+cG8xT/cyOvp9wwN3IxfdYgpHs8CE7Nq2PKX2lNaEXw==}+ engines: {node: '>=14'}+'@ioredis/[email protected]':resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}@@ -1553,6 +1572,10 @@ packages:'@supabase/[email protected]':resolution: {integrity: sha512-fouCwL1OxqftOwLNgdDUPlNnFuCnt30nS4kLcnTpe6NYKn1PmjxRRBFmKscgHs6FjWyU+32ZG4uBJ29+/BWiDw==}+ '@tootallnate/[email protected]':+ resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}+ engines: {node: '>= 10'}+'@tootallnate/[email protected]':resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==}@@ -1586,6 +1609,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}+ '@types/[email protected]':+ resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==}+'@types/[email protected]':resolution: {integrity: sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==}@@ -1673,6 +1699,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}+ '@types/[email protected]':+ resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==}+'@types/[email protected]':resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==}@@ -1691,6 +1720,9 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-137ypx2lk/wTQbW6An6safu9hXmajAifU/s7szAHLN/FeIm5w7yR0Wkl9fdJMRSHwOn4HLAI0DaB2TOORuhPDg==}+ '@types/[email protected]':+ resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==}+'@types/[email protected]':resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==}@@ -1843,6 +1875,10 @@ packages:resolution: {integrity: sha512-JIUwuFakO+jHjxyp4YgSiKXSZeC0U+R1jR94bXWBcVlFRBycqXlb+kH9JHxBGcxnVuSqx5bnn0Qz9xtSeKOjiA==}+ resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==}+ engines: {node: '>=8'}+resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}@@ -1853,6 +1889,9 @@ packages:resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==}+ resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}+resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==}@@ -1933,6 +1972,9 @@ packages:resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==}engines: {node: '>=10.0.0'}+ resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==}+resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}@@ -1986,6 +2028,9 @@ packages:resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}+ resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}+resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}@@ -2006,6 +2051,10 @@ packages:resolution: {integrity: sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==}engines: {node: '>=10.6.0'}+ resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}engines: {node: '>= 0.4'}@@ -2367,9 +2416,19 @@ packages:resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==}+ resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}+ engines: {node: '>= 0.4'}++ resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==}+resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}+ resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}+resolution: {integrity: sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==}engines: {node: '>=14'}@@ -2424,10 +2483,22 @@ packages:resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}+ engines: {node: '>= 0.4'}++ resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}engines: {node: '>=6'}@@ -2508,6 +2579,9 @@ packages:resolution: {integrity: sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==}engines: {node: '>= 0.10.0'}+ resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}+resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==}engines: {node: '>= 10.17.0'}@@ -2587,6 +2661,10 @@ packages:resolution: {integrity: sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==}engines: {node: '>= 18'}+ resolution: {integrity: sha512-XHIrMD0NpDrNM/Ckf7XJiBbLl57KEhT3+i3yY+eWm+cqYZJQTZrKo8Y8AWKnuV5GT4scfuUGt9LzNoIx3dU1nQ==}+ engines: {node: '>= 0.12'}+resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}engines: {node: '>= 6'}@@ -2633,6 +2711,14 @@ packages:resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}+ resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==}+ engines: {node: '>=14'}++ resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==}+ engines: {node: '>=14'}+resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}engines: {node: '>= 4'}@@ -2649,10 +2735,18 @@ packages:resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==}engines: {node: '>=8.0.0'}+ resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==}engines: {node: '>=8'}@@ -2682,9 +2776,21 @@ packages:resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==}engines: {node: '>=4'}+ resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==}+ engines: {node: '>=14'}++ resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==}+ engines: {node: '>=14'}+resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}+ resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-O9iCL8MqGR0Oe9wTh0YftzIbysypNQmS5a5JG3cB3M4LMYjlAVvNnf8LUzVY9MrI7tj+YLY356uHtO2lLX2HpA==}engines: {node: '>=12'}@@ -2692,6 +2798,10 @@ packages:resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}+ resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==}+ engines: {node: '>=14.0.0'}+resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}engines: {node: '>=4'}@@ -2711,6 +2821,14 @@ packages:resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==}engines: {node: '>= 0.4'}+ resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}+ engines: {node: '>= 0.4'}++ resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}engines: {node: '>= 0.4'}@@ -2731,6 +2849,9 @@ packages:resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==}engines: {node: '>=18'}+ resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==}+resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}@@ -2745,6 +2866,10 @@ packages:resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}engines: {node: '>= 0.8'}+ resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==}+ engines: {node: '>= 6'}+resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}engines: {node: '>= 14'}@@ -3111,6 +3236,9 @@ packages:engines: {node: '>=4'}hasBin: true+ resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==}+resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}@@ -3140,6 +3268,12 @@ packages:resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==}+ resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==}++ resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==}+resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}engines: {node: '>=12.0.0'}@@ -3258,6 +3392,10 @@ packages:engines: {node: '>= 18'}hasBin: true+ resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}+ engines: {node: '>= 0.4'}+resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}@@ -3304,6 +3442,11 @@ packages:engines: {node: '>=4.0.0'}hasBin: true+ resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}+ engines: {node: '>=10.0.0'}+ hasBin: true+resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==}engines: {node: '>=6'}@@ -3896,6 +4039,14 @@ packages:resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}hasBin: true+ resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==}+ engines: {node: '>=14'}++ resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}+ engines: {node: '>= 4'}+resolution: {integrity: sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==}engines: {node: '>=14.18'}@@ -4075,6 +4226,12 @@ packages:resolution: {integrity: sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==}engines: {node: '>=0.10.0'}+ resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==}++ resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}+resolution: {integrity: sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==}@@ -4123,6 +4280,9 @@ packages:resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}+ resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==}+resolution: {integrity: sha512-h2J6kKEikXnZyurUcCYg215qkQpINOhdWkiclHcWAuVeqXsNrfrYaf1s0qbbcdRyMtrVW48I+VdVTw71Cnn20Q==}engines: {npm: '>=8'}@@ -4181,6 +4341,10 @@ packages:resolution: {integrity: sha512-hctwP0Nb4AB60bj8WQgRYaMOuJYRAPMGiQUAotms5igN8ppfQM+IvjQ5HcKu1MaZh2Wy2KWVTe563Yj8dfc14w==}engines: {node: '>=18'}+ resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==}+ engines: {node: '>=14'}+resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==}engines: {node: '>=8'}@@ -4305,13 +4469,8 @@ packages:resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==}engines: {node: '>= 0.6'}- resolution: {integrity: sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==}- engines: {node: '>=14.17'}- hasBin: true-- resolution: {integrity: sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==}+ resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==}engines: {node: '>=14.17'}hasBin: true@@ -4385,6 +4544,10 @@ packages:resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}hasBin: true+ resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}+ hasBin: true+resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}hasBin: true@@ -5423,6 +5586,36 @@ snapshots:'@dqbd/[email protected]': {}+ '@google-cloud/[email protected]':+ dependencies:+ arrify: 2.0.1+ extend: 3.0.2++ '@google-cloud/[email protected]': {}++ '@google-cloud/[email protected]': {}++ '@google-cloud/[email protected]([email protected])':+ dependencies:+ '@google-cloud/paginator': 5.0.2+ '@google-cloud/projectify': 4.0.0+ '@google-cloud/promisify': 4.0.0+ abort-controller: 3.0.0+ async-retry: 1.3.3+ duplexify: 4.1.3+ fast-xml-parser: 4.4.1+ gaxios: 6.7.1([email protected])+ google-auth-library: 9.15.1([email protected])+ html-entities: 2.6.0+ mime: 3.0.0+ p-limit: 3.1.0+ retry-request: 7.0.2([email protected])+ teeny-request: 9.0.0([email protected])+ uuid: 8.3.2+ transitivePeerDependencies:+ - encoding+ - supports-color+'@ioredis/[email protected]': {}'@isaacs/[email protected]':@@ -5457,7 +5650,7 @@ snapshots:jest-util: 29.7.0slash: 3.0.0dependencies:'@jest/console': 29.7.0'@jest/reporters': 29.7.0@@ -5471,7 +5664,7 @@ snapshots:exit: 0.1.2graceful-fs: 4.2.11jest-changed-files: 29.7.0- jest-config: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ jest-config: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))jest-haste-map: 29.7.0jest-message-util: 29.7.0jest-regex-util: 29.6.3@@ -6530,6 +6723,8 @@ snapshots:- bufferutil- utf-8-validate+ '@tootallnate/[email protected]': {}+'@tootallnate/[email protected]': {}'@tsconfig/[email protected]': {}@@ -6568,6 +6763,8 @@ snapshots:'@types/connect': 3.4.38'@types/node': 20.14.1+ '@types/[email protected]': {}+'@types/[email protected]':dependencies:'@types/node': 20.14.1@@ -6670,6 +6867,13 @@ snapshots:'@types/[email protected]': {}+ '@types/[email protected]':+ dependencies:+ '@types/caseless': 0.12.5+ '@types/node': 20.14.1+ '@types/tough-cookie': 4.0.5+ form-data: 2.5.3+'@types/[email protected]':dependencies:'@types/mime': 1.3.5@@ -6697,6 +6901,8 @@ snapshots:'@types/methods': 1.1.4'@types/superagent': 8.1.9+ '@types/[email protected]': {}+'@types/[email protected]': {}'@types/[email protected]': {}@@ -6832,6 +7038,8 @@ snapshots:+ [email protected]: {}+@@ -6842,6 +7050,10 @@ snapshots:dependencies:tslib: 2.6.3+ dependencies:+ retry: 0.13.1+dependencies:lodash: 4.17.21@@ -6961,6 +7173,8 @@ snapshots:+ [email protected]: {}+dependencies:cmd-shim: 6.0.3@@ -7027,6 +7241,8 @@ snapshots:+ [email protected]: {}+@@ -7055,6 +7271,11 @@ snapshots:+ dependencies:+ es-errors: 1.3.0+ function-bind: 1.1.2+dependencies:es-define-property: 1.0.0@@ -7234,22 +7455,22 @@ snapshots:dependencies:env-paths: 2.2.1import-fresh: 3.3.0js-yaml: 4.1.0parse-json: 5.2.0optionalDependencies:- typescript: 5.4.5+ typescript: 5.8.3- [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):+ [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):dependencies:'@jest/types': 29.6.3chalk: 4.1.2exit: 0.1.2graceful-fs: 4.2.11- jest-config: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ jest-config: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))jest-util: 29.7.0prompts: 2.4.2transitivePeerDependencies:@@ -7406,8 +7627,25 @@ snapshots:dependencies:underscore: 1.13.6+ dependencies:+ call-bind-apply-helpers: 1.0.2+ es-errors: 1.3.0+ gopd: 1.2.0++ dependencies:+ end-of-stream: 1.4.4+ inherits: 2.0.4+ readable-stream: 3.6.2+ stream-shift: 1.0.3++ dependencies:+ safe-buffer: 5.2.1+dependencies:'@one-ini/wasm': 0.1.1@@ -7453,8 +7691,21 @@ snapshots:dependencies:get-intrinsic: 1.2.4+ [email protected]: {}++ dependencies:+ es-errors: 1.3.0++ dependencies:+ es-errors: 1.3.0+ get-intrinsic: 1.3.0+ has-tostringtag: 1.0.2+ hasown: 2.0.2+@@ -7557,6 +7808,8 @@ snapshots:transitivePeerDependencies:- supports-color+ [email protected]: {}+dependencies:debug: 4.3.5@@ -7636,6 +7889,14 @@ snapshots:+ dependencies:+ asynckit: 0.4.0+ combined-stream: 1.0.8+ es-set-tostringtag: 2.1.0+ mime-types: 2.1.35+ safe-buffer: 5.2.1+dependencies:asynckit: 0.4.0@@ -7683,6 +7944,26 @@ snapshots:+ dependencies:+ extend: 3.0.2+ https-proxy-agent: 7.0.6+ is-stream: 2.0.1+ node-fetch: 2.7.0([email protected])+ uuid: 9.0.1+ transitivePeerDependencies:+ - encoding+ - supports-color++ dependencies:+ gaxios: 6.7.1([email protected])+ google-logging-utils: 0.0.2+ json-bigint: 1.0.0+ transitivePeerDependencies:+ - encoding+ - supports-color+@@ -7697,8 +7978,26 @@ snapshots:has-symbols: 1.0.3hasown: 2.0.2+ dependencies:+ call-bind-apply-helpers: 1.0.2+ es-define-property: 1.0.1+ es-errors: 1.3.0+ es-object-atoms: 1.1.1+ function-bind: 1.1.2+ get-proto: 1.0.1+ gopd: 1.2.0+ has-symbols: 1.1.0+ hasown: 2.0.2+ math-intrinsics: 1.1.0++ dependencies:+ dunder-proto: 1.0.1+ es-object-atoms: 1.1.1+dependencies:pump: 3.0.0@@ -7738,16 +8037,40 @@ snapshots:+ dependencies:+ base64-js: 1.5.1+ ecdsa-sig-formatter: 1.0.11+ gaxios: 6.7.1([email protected])+ gcp-metadata: 6.1.1([email protected])+ gtoken: 7.1.0([email protected])+ jws: 4.0.0+ transitivePeerDependencies:+ - encoding+ - supports-color++ [email protected]: {}+dependencies:get-intrinsic: 1.2.4+ [email protected]: {}+dependencies:array-keyed-map: 2.1.3+ dependencies:+ gaxios: 6.7.1([email protected])+ jws: 4.0.0+ transitivePeerDependencies:+ - encoding+ - supports-color+@@ -7760,6 +8083,12 @@ snapshots:+ [email protected]: {}++ dependencies:+ has-symbols: 1.0.3+dependencies:function-bind: 1.1.2@@ -7776,6 +8105,8 @@ snapshots:dependencies:whatwg-encoding: 3.1.1+ [email protected]: {}+@@ -7801,6 +8132,14 @@ snapshots:statuses: 2.0.1toidentifier: 1.0.1+ dependencies:+ '@tootallnate/once': 2.0.0+ agent-base: 6.0.2+ debug: 4.3.5+ transitivePeerDependencies:+ - supports-color+dependencies:agent-base: 7.1.1@@ -8073,16 +8412,16 @@ snapshots:- babel-plugin-macros- supports-color- [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):+ [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):dependencies:'@jest/test-result': 29.7.0'@jest/types': 29.6.3chalk: 4.1.2- create-jest: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ create-jest: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))exit: 0.1.2import-local: 3.1.0- jest-config: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ jest-config: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))jest-util: 29.7.0jest-validate: 29.7.0yargs: 17.7.2@@ -8092,7 +8431,7 @@ snapshots:- supports-color- ts-node- [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):+ [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):dependencies:'@babel/core': 7.24.6'@jest/test-sequencer': 29.7.0@@ -8118,7 +8457,7 @@ snapshots:strip-json-comments: 3.1.1optionalDependencies:'@types/node': 20.14.1- ts-node: 10.9.2(@types/[email protected])([email protected])+ ts-node: 10.9.2(@types/[email protected])([email protected])transitivePeerDependencies:- babel-plugin-macros- supports-color@@ -8345,12 +8684,12 @@ snapshots:merge-stream: 2.0.0supports-color: 8.1.1- [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):+ [email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])):dependencies:'@jest/types': 29.6.3import-local: 3.1.0- jest-cli: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ jest-cli: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))transitivePeerDependencies:- '@types/node'- babel-plugin-macros@@ -8414,6 +8753,10 @@ snapshots:+ dependencies:+ bignumber.js: 9.2.0+@@ -8443,6 +8786,17 @@ snapshots:readable-stream: 2.3.8setimmediate: 1.0.5+ dependencies:+ buffer-equal-constant-time: 1.0.1+ ecdsa-sig-formatter: 1.0.11+ safe-buffer: 5.2.1++ dependencies:+ jwa: 2.0.0+ safe-buffer: 5.2.1+@@ -8556,6 +8910,8 @@ snapshots:+ [email protected]: {}+dependencies:charenc: 0.0.2@@ -8589,6 +8945,8 @@ snapshots:+ [email protected]: {}+@@ -9036,7 +9394,7 @@ snapshots:csv-parse: 5.5.6gpt3-tokenizer: 1.1.5openai: 3.3.0- typescript: 5.8.2+ typescript: 5.8.3uuid: 9.0.1zod: 3.24.2transitivePeerDependencies:@@ -9090,10 +9448,10 @@ snapshots:- supports-color- utf-8-validatedependencies:'@puppeteer/browsers': 2.2.3- cosmiconfig: 9.0.0([email protected])+ cosmiconfig: 9.0.0([email protected])devtools-protocol: 0.0.1299070puppeteer-core: 22.12.1transitivePeerDependencies:@@ -9234,6 +9592,17 @@ snapshots:path-parse: 1.0.7supports-preserve-symlinks-flag: 1.0.0+ dependencies:+ '@types/request': 2.48.12+ extend: 3.0.2+ teeny-request: 9.0.0([email protected])+ transitivePeerDependencies:+ - encoding+ - supports-color++ [email protected]: {}+dependencies:glob: 10.4.2@@ -9407,6 +9776,12 @@ snapshots:+ dependencies:+ stubs: 3.0.0++ [email protected]: {}+dependencies:fast-fifo: 1.3.2@@ -9461,6 +9836,8 @@ snapshots:+ [email protected]: {}+dependencies:bin-links: 4.0.4@@ -9541,6 +9918,17 @@ snapshots:mkdirp: 3.0.1yallist: 5.0.0+ dependencies:+ http-proxy-agent: 5.0.0+ https-proxy-agent: 5.0.1+ node-fetch: 2.7.0([email protected])+ stream-events: 1.0.5+ uuid: 9.0.1+ transitivePeerDependencies:+ - encoding+ - supports-color+dependencies:'@istanbuljs/schema': 0.1.3@@ -9591,17 +9979,17 @@ snapshots:- [email protected](@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected]):+ [email protected](@babel/[email protected])(@jest/[email protected])(@jest/[email protected])([email protected](@babel/[email protected]))([email protected](@types/[email protected])([email protected](@types/[email protected])([email protected])))([email protected]):dependencies:bs-logger: 0.2.6fast-json-stable-stringify: 2.1.0- jest: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))+ jest: 29.7.0(@types/[email protected])([email protected](@types/[email protected])([email protected]))jest-util: 29.7.0json5: 2.2.3lodash.memoize: 4.1.2make-error: 1.3.6semver: 7.6.2- typescript: 5.4.5+ typescript: 5.8.3yargs-parser: 21.1.1optionalDependencies:'@babel/core': 7.24.6@@ -9609,7 +9997,7 @@ snapshots:'@jest/types': 29.6.3babel-jest: 29.7.0(@babel/[email protected])dependencies:'@cspotcode/source-map-support': 0.8.1'@tsconfig/node10': 1.0.11@@ -9623,7 +10011,7 @@ snapshots:create-require: 1.1.1diff: 4.0.2make-error: 1.3.6- typescript: 5.4.5+ typescript: 5.8.3v8-compile-cache-lib: 3.0.1yn: 3.1.1@@ -9646,9 +10034,7 @@ snapshots:media-typer: 0.3.0mime-types: 2.1.35- [email protected]: {}-- [email protected]: {}+ [email protected]: {}[email protected](@babel/[email protected]):dependencies:@@ -9709,6 +10095,8 @@ snapshots:+ [email protected]: {}+diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex ac41b6e83..5432687f8 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -1,10 +1,10 @@-import { ExtractorOptions } from "./../../lib/entities";import { supabase_service } from "../supabase";import { FirecrawlJob } from "../../types";import { posthog } from "../posthog";import "dotenv/config";import { logger } from "../../lib/logger";import { configDotenv } from "dotenv";+import { Storage } from "@google-cloud/storage";configDotenv();function cleanOfNull<T>(x: T): T {@@ -21,6 +21,43 @@ function cleanOfNull<T>(x: T): T {}}++async function saveJobToGCS(job: FirecrawlJob, bucketName: string): Promise<void> {+ try {+ const storage = new Storage();+ const bucket = storage.bucket(bucketName);+ const blob = bucket.file(`${job.job_id}.json`);+ await blob.save(JSON.stringify(job.docs), {+ contentType: "application/json",+ });+ await blob.setMetadata({+ metadata: {+ job_id: job.job_id ?? null,+ success: job.success,+ message: job.message ?? null,+ num_docs: job.num_docs,+ time_taken: job.time_taken,+ team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_"))? null : job.team_id,+ mode: job.mode,+ url: job.url,+ crawler_options: job.crawlerOptions,+ page_options: job.scrapeOptions,+ origin: job.origin,+ num_tokens: job.num_tokens ?? null,+ retry: !!job.retry,+ crawl_id: job.crawl_id ?? null,+ tokens_billed: job.tokens_billed ?? null,+ },+ })+ } catch (error) {+ logger.error(`Error saving job to GCS`, {+ error,+ scrapeId: job.job_id,+ jobId: job.job_id,+ });+ }+}+async function indexJob(job: FirecrawlJob): Promise<void> {try {if (job.mode !== "single_urls" && job.mode !== "scrape") {@@ -108,6 +145,10 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {indexJob(job);}+ if (process.env.GCS_BUCKET_NAME) {+ await saveJobToGCS(job, process.env.GCS_BUCKET_NAME);+ }+if (force) {let i = 0,done = false;