Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(log_job): start saving jobs to GCS (#1424)

Gergő Móriczbd1c1b0012e8
diff --git a/apps/api/package.json b/apps/api/package.json
index 3feec21f8..357914831 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -48,7 +48,7 @@
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
- "typescript": "^5.4.2"
+ "typescript": "^5.8.3"
},
"dependencies": {
"@ai-sdk/openai": "^1.1.13",
@@ -59,6 +59,7 @@
"@bull-board/express": "^5.20.5",
"@devil7softwares/pos": "^1.0.2",
"@dqbd/tiktoken": "^1.0.17",
+ "@google-cloud/storage": "^7.16.0",
"@nangohq/node": "^0.40.8",
"@pinecone-database/pinecone": "^4.0.0",
"@sentry/cli": "^2.33.1",
diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yaml
index fdd8aa142..508fb7cd0 100644
--- a/apps/api/pnpm-lock.yaml
+++ b/apps/api/pnpm-lock.yaml
@@ -32,6 +32,9 @@ importers:
'@dqbd/tiktoken':
specifier: ^1.0.17
version: 1.0.17
+ '@google-cloud/storage':
+ specifier: ^7.16.0
+ version: 7.16.0([email protected])
'@nangohq/node':
specifier: ^0.40.8
version: 0.40.8
@@ -196,7 +199,7 @@ importers:
version: 0.0.10
puppeteer:
specifier: ^22.12.1
- version: 22.12.1([email protected])
+ version: 22.12.1([email protected])
rate-limiter-flexible:
specifier: 2.4.2
version: 2.4.2
@@ -296,7 +299,7 @@ importers:
version: 6.0.2
jest:
specifier: ^29.6.3
jest-fetch-mock:
specifier: ^3.0.3
version: 3.0.3([email protected])
@@ -311,13 +314,13 @@ importers:
version: 6.3.4
ts-jest:
specifier: ^29.1.1
ts-node:
specifier: ^10.9.1
- version: 10.9.2(@types/[email protected])([email protected])
+ version: 10.9.2(@types/[email protected])([email protected])
typescript:
- specifier: ^5.4.2
- version: 5.4.5
+ specifier: ^5.8.3
+ version: 5.8.3
packages:
@@ -767,6 +770,22 @@ packages:
resolution: {integrity: sha512-v2gz0V6DiuR2TsALM32TkBThf6LdjLbxe6HS/nx9/KJxuDX0Z7SGX7N7PvQfqIvRyus42jI9poVUqezc/j/aQw==}
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}
+ engines: {node: '>=14.0.0'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==}
+ engines: {node: '>=14.0.0'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==}
+ engines: {node: '>=14'}
+
+ '@google-cloud/[email protected]':
+ resolution: {integrity: sha512-7/5LRgykyOfQENcm6hDKP8SX/u9XxE5YOiWOkgkwcoO+cG8xT/cyOvp9wwN3IxfdYgpHs8CE7Nq2PKX2lNaEXw==}
+ engines: {node: '>=14'}
+
'@ioredis/[email protected]':
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
@@ -1553,6 +1572,10 @@ packages:
'@supabase/[email protected]':
resolution: {integrity: sha512-fouCwL1OxqftOwLNgdDUPlNnFuCnt30nS4kLcnTpe6NYKn1PmjxRRBFmKscgHs6FjWyU+32ZG4uBJ29+/BWiDw==}
+ '@tootallnate/[email protected]':
+ resolution: {integrity: sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==}
+ engines: {node: '>= 10'}
+
'@tootallnate/[email protected]':
resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==}
@@ -1586,6 +1609,9 @@ packages:
resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==}
+
resolution: {integrity: sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==}
@@ -1673,6 +1699,9 @@ packages:
resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==}
+
resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==}
@@ -1691,6 +1720,9 @@ packages:
resolution: {integrity: sha512-137ypx2lk/wTQbW6An6safu9hXmajAifU/s7szAHLN/FeIm5w7yR0Wkl9fdJMRSHwOn4HLAI0DaB2TOORuhPDg==}
+ '@types/[email protected]':
+ resolution: {integrity: sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==}
+
resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==}
@@ -1843,6 +1875,10 @@ packages:
resolution: {integrity: sha512-JIUwuFakO+jHjxyp4YgSiKXSZeC0U+R1jR94bXWBcVlFRBycqXlb+kH9JHxBGcxnVuSqx5bnn0Qz9xtSeKOjiA==}
+ resolution: {integrity: sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==}
+ engines: {node: '>=8'}
+
resolution: {integrity: sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==}
@@ -1853,6 +1889,9 @@ packages:
resolution: {integrity: sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA==}
+ resolution: {integrity: sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==}
+
resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==}
@@ -1933,6 +1972,9 @@ packages:
resolution: {integrity: sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==}
engines: {node: '>=10.0.0'}
+ resolution: {integrity: sha512-JocpCSOixzy5XFJi2ub6IMmV/G9i8Lrm2lZvwBv9xPdglmZM0ufDVBbjbrfU/zuLvBfD7Bv2eYxz9i+OHTgkew==}
+
resolution: {integrity: sha512-cMtq4W5ZsEwcutJrVId+a/tjt8GSbS+h0oNkdl6+6rBuEv8Ot33Bevj5KPm40t309zuhVic8NjpuL42QCiJWWA==}
engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0}
@@ -1986,6 +2028,9 @@ packages:
resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
+ resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}
+
resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
@@ -2006,6 +2051,10 @@ packages:
resolution: {integrity: sha512-KJ/Dmo1lDDhmW2XDPMo+9oiy/CeqosPguPCrgcVzKyZrL6pM1gU2GmPY/xo6OQPTUaA/c0kwHuywB4E6nmT9ww==}
engines: {node: '>=10.6.0'}
+ resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}
engines: {node: '>= 0.4'}
@@ -2367,9 +2416,19 @@ packages:
resolution: {integrity: sha512-wkctla1O6VfP89gQ+J/yDesM0S7B7XLXjKGzXxMDVFg7uEn706niAtyYovKbyq1oT9YwDcly721/iUWoc8MVRg==}
+ resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==}
+ engines: {node: '>= 0.4'}
+
+ resolution: {integrity: sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==}
+
resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==}
+ resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==}
+
resolution: {integrity: sha512-L9Qe08KWTlqYMVvMcTIvMAdl1cDUubzRNYL+WfA4bLDMHe4nemKkpmYzkznE1FwLKu0EEmy6obgQKzMJrg4x9Q==}
engines: {node: '>=14'}
@@ -2424,10 +2483,22 @@ packages:
resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==}
+ engines: {node: '>= 0.4'}
+
+ resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==}
engines: {node: '>=6'}
@@ -2508,6 +2579,9 @@ packages:
resolution: {integrity: sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==}
engines: {node: '>= 0.10.0'}
+ resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
+
resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==}
engines: {node: '>= 10.17.0'}
@@ -2587,6 +2661,10 @@ packages:
resolution: {integrity: sha512-KQVhvhK8ZkWzxKxOr56CPulAhH3dobtuQ4+hNQ+HekH/Wp5gSOafqRAeTphQUJAIk0GBvHZgJ2ZGRWd5kphMuw==}
engines: {node: '>= 18'}
+ resolution: {integrity: sha512-XHIrMD0NpDrNM/Ckf7XJiBbLl57KEhT3+i3yY+eWm+cqYZJQTZrKo8Y8AWKnuV5GT4scfuUGt9LzNoIx3dU1nQ==}
+ engines: {node: '>= 0.12'}
+
resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==}
engines: {node: '>= 6'}
@@ -2633,6 +2711,14 @@ packages:
resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
+ resolution: {integrity: sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ==}
+ engines: {node: '>=14'}
+
+ resolution: {integrity: sha512-a4tiq7E0/5fTjxPAaH4jpjkSv/uCaU2p5KC6HVGrvl0cDjA8iBZv4vv1gyzlmK0ZUKqwpOyQMKzZQe3lTit77A==}
+ engines: {node: '>=14'}
+
resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==}
engines: {node: '>= 4'}
@@ -2649,10 +2735,18 @@ packages:
resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==}
engines: {node: '>=8.0.0'}
+ resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==}
engines: {node: '>=8'}
@@ -2682,9 +2776,21 @@ packages:
resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==}
engines: {node: '>=4'}
+ resolution: {integrity: sha512-Jb6Z0+nvECVz+2lzSMt9u98UsoakXxA2HGHMCxh+so3n90XgYWkq5dur19JAJV7ONiJY22yBTyJB1TSkvPq9Ng==}
+ engines: {node: '>=14'}
+
+ resolution: {integrity: sha512-NEgUnEcBiP5HrPzufUkBzJOD/Sxsco3rLNo1F1TNf7ieU8ryUzBhqba8r756CjLX7rn3fHl6iLEwPYuqpoKgQQ==}
+ engines: {node: '>=14'}
+
resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==}
+ resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-O9iCL8MqGR0Oe9wTh0YftzIbysypNQmS5a5JG3cB3M4LMYjlAVvNnf8LUzVY9MrI7tj+YLY356uHtO2lLX2HpA==}
engines: {node: '>=12'}
@@ -2692,6 +2798,10 @@ packages:
resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
+ resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==}
+ engines: {node: '>=14.0.0'}
+
resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==}
engines: {node: '>=4'}
@@ -2711,6 +2821,14 @@ packages:
resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==}
engines: {node: '>= 0.4'}
+ resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==}
+ engines: {node: '>= 0.4'}
+
+ resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==}
engines: {node: '>= 0.4'}
@@ -2731,6 +2849,9 @@ packages:
resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-kig+rMn/QOVRvr7c86gQ8lWXq+Hkv6CbAH1hLu+RG338StTpE8Z0b44SDVaqVu7HGKf27frdmUYEs9hTUX/cLQ==}
+
resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==}
@@ -2745,6 +2866,10 @@ packages:
resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
engines: {node: '>= 0.8'}
+ resolution: {integrity: sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==}
+ engines: {node: '>= 6'}
+
resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==}
engines: {node: '>= 14'}
@@ -3111,6 +3236,9 @@ packages:
engines: {node: '>=4'}
hasBin: true
+ resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==}
+
resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==}
@@ -3140,6 +3268,12 @@ packages:
resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==}
+ resolution: {integrity: sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==}
+
+ resolution: {integrity: sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==}
+
resolution: {integrity: sha512-C3iHfuGUXK2u8/ipq9LfjFfXFxAZMQJJq7vLS45r3D9Y2xQ/m4S8zaR4zMLFWh9AsNPXmcFfUDhTEO8UIC/V6Q==}
engines: {node: '>=12.0.0'}
@@ -3258,6 +3392,10 @@ packages:
engines: {node: '>= 18'}
hasBin: true
+ resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
+ engines: {node: '>= 0.4'}
+
resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==}
@@ -3304,6 +3442,11 @@ packages:
engines: {node: '>=4.0.0'}
hasBin: true
+ resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==}
+ engines: {node: '>=10.0.0'}
+ hasBin: true
+
resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==}
engines: {node: '>=6'}
@@ -3896,6 +4039,14 @@ packages:
resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==}
hasBin: true
+ resolution: {integrity: sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==}
+ engines: {node: '>=14'}
+
+ resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==}
+ engines: {node: '>= 4'}
+
resolution: {integrity: sha512-nV6YcJo5wbLW77m+8KjH8aB/7/rxQy9SZ0HY5shnwULfS+9nmTtVXAJET5NdZmCzA4fPI/Hm1wo/Po/4mopOdg==}
engines: {node: '>=14.18'}
@@ -4075,6 +4226,12 @@ packages:
resolution: {integrity: sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==}
engines: {node: '>=0.10.0'}
+ resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==}
+
+ resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==}
+
resolution: {integrity: sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ==}
@@ -4123,6 +4280,9 @@ packages:
resolution: {integrity: sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==}
+ resolution: {integrity: sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==}
+
resolution: {integrity: sha512-h2J6kKEikXnZyurUcCYg215qkQpINOhdWkiclHcWAuVeqXsNrfrYaf1s0qbbcdRyMtrVW48I+VdVTw71Cnn20Q==}
engines: {npm: '>=8'}
@@ -4181,6 +4341,10 @@ packages:
resolution: {integrity: sha512-hctwP0Nb4AB60bj8WQgRYaMOuJYRAPMGiQUAotms5igN8ppfQM+IvjQ5HcKu1MaZh2Wy2KWVTe563Yj8dfc14w==}
engines: {node: '>=18'}
+ resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==}
+ engines: {node: '>=14'}
+
resolution: {integrity: sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==}
engines: {node: '>=8'}
@@ -4305,13 +4469,8 @@ packages:
resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==}
engines: {node: '>= 0.6'}
- resolution: {integrity: sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==}
- engines: {node: '>=14.17'}
- hasBin: true
-
- resolution: {integrity: sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==}
+ resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==}
engines: {node: '>=14.17'}
hasBin: true
@@ -4385,6 +4544,10 @@ packages:
resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==}
hasBin: true
+ resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==}
+ hasBin: true
+
resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==}
hasBin: true
@@ -5423,6 +5586,36 @@ snapshots:
'@dqbd/[email protected]': {}
+ '@google-cloud/[email protected]':
+ dependencies:
+ arrify: 2.0.1
+ extend: 3.0.2
+
+ '@google-cloud/[email protected]': {}
+
+ '@google-cloud/[email protected]': {}
+
+ dependencies:
+ '@google-cloud/paginator': 5.0.2
+ '@google-cloud/projectify': 4.0.0
+ '@google-cloud/promisify': 4.0.0
+ abort-controller: 3.0.0
+ async-retry: 1.3.3
+ duplexify: 4.1.3
+ fast-xml-parser: 4.4.1
+ gaxios: 6.7.1([email protected])
+ google-auth-library: 9.15.1([email protected])
+ html-entities: 2.6.0
+ mime: 3.0.0
+ p-limit: 3.1.0
+ retry-request: 7.0.2([email protected])
+ teeny-request: 9.0.0([email protected])
+ uuid: 8.3.2
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
'@ioredis/[email protected]': {}
'@isaacs/[email protected]':
@@ -5457,7 +5650,7 @@ snapshots:
jest-util: 29.7.0
slash: 3.0.0
dependencies:
'@jest/console': 29.7.0
'@jest/reporters': 29.7.0
@@ -5471,7 +5664,7 @@ snapshots:
exit: 0.1.2
graceful-fs: 4.2.11
jest-changed-files: 29.7.0
jest-haste-map: 29.7.0
jest-message-util: 29.7.0
jest-regex-util: 29.6.3
@@ -6530,6 +6723,8 @@ snapshots:
- bufferutil
- utf-8-validate
+ '@tootallnate/[email protected]': {}
+
'@tootallnate/[email protected]': {}
'@tsconfig/[email protected]': {}
@@ -6568,6 +6763,8 @@ snapshots:
'@types/connect': 3.4.38
'@types/node': 20.14.1
+ '@types/[email protected]': {}
+
dependencies:
'@types/node': 20.14.1
@@ -6670,6 +6867,13 @@ snapshots:
'@types/[email protected]': {}
+ '@types/[email protected]':
+ dependencies:
+ '@types/caseless': 0.12.5
+ '@types/node': 20.14.1
+ '@types/tough-cookie': 4.0.5
+ form-data: 2.5.3
+
dependencies:
'@types/mime': 1.3.5
@@ -6697,6 +6901,8 @@ snapshots:
'@types/methods': 1.1.4
'@types/superagent': 8.1.9
+ '@types/[email protected]': {}
+
'@types/[email protected]': {}
'@types/[email protected]': {}
@@ -6832,6 +7038,8 @@ snapshots:
+
@@ -6842,6 +7050,10 @@ snapshots:
dependencies:
tslib: 2.6.3
+ dependencies:
+ retry: 0.13.1
+
dependencies:
lodash: 4.17.21
@@ -6961,6 +7173,8 @@ snapshots:
+
dependencies:
cmd-shim: 6.0.3
@@ -7027,6 +7241,8 @@ snapshots:
+
@@ -7055,6 +7271,11 @@ snapshots:
+ dependencies:
+ es-errors: 1.3.0
+ function-bind: 1.1.2
+
dependencies:
es-define-property: 1.0.0
@@ -7234,22 +7455,22 @@ snapshots:
dependencies:
env-paths: 2.2.1
import-fresh: 3.3.0
js-yaml: 4.1.0
parse-json: 5.2.0
optionalDependencies:
- typescript: 5.4.5
+ typescript: 5.8.3
dependencies:
'@jest/types': 29.6.3
chalk: 4.1.2
exit: 0.1.2
graceful-fs: 4.2.11
jest-util: 29.7.0
prompts: 2.4.2
transitivePeerDependencies:
@@ -7406,8 +7627,25 @@ snapshots:
dependencies:
underscore: 1.13.6
+ dependencies:
+ call-bind-apply-helpers: 1.0.2
+ es-errors: 1.3.0
+ gopd: 1.2.0
+
+ dependencies:
+ end-of-stream: 1.4.4
+ inherits: 2.0.4
+ readable-stream: 3.6.2
+ stream-shift: 1.0.3
+
+ dependencies:
+ safe-buffer: 5.2.1
+
dependencies:
'@one-ini/wasm': 0.1.1
@@ -7453,8 +7691,21 @@ snapshots:
dependencies:
get-intrinsic: 1.2.4
+
+ dependencies:
+ es-errors: 1.3.0
+
+ dependencies:
+ es-errors: 1.3.0
+ get-intrinsic: 1.3.0
+ has-tostringtag: 1.0.2
+ hasown: 2.0.2
+
@@ -7557,6 +7808,8 @@ snapshots:
transitivePeerDependencies:
- supports-color
+
dependencies:
debug: 4.3.5
@@ -7636,6 +7889,14 @@ snapshots:
+ dependencies:
+ asynckit: 0.4.0
+ combined-stream: 1.0.8
+ es-set-tostringtag: 2.1.0
+ mime-types: 2.1.35
+ safe-buffer: 5.2.1
+
dependencies:
asynckit: 0.4.0
@@ -7683,6 +7944,26 @@ snapshots:
+ dependencies:
+ extend: 3.0.2
+ https-proxy-agent: 7.0.6
+ is-stream: 2.0.1
+ node-fetch: 2.7.0([email protected])
+ uuid: 9.0.1
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
+ dependencies:
+ gaxios: 6.7.1([email protected])
+ google-logging-utils: 0.0.2
+ json-bigint: 1.0.0
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
@@ -7697,8 +7978,26 @@ snapshots:
has-symbols: 1.0.3
hasown: 2.0.2
+ dependencies:
+ call-bind-apply-helpers: 1.0.2
+ es-define-property: 1.0.1
+ es-errors: 1.3.0
+ es-object-atoms: 1.1.1
+ function-bind: 1.1.2
+ get-proto: 1.0.1
+ gopd: 1.2.0
+ has-symbols: 1.1.0
+ hasown: 2.0.2
+ math-intrinsics: 1.1.0
+
+ dependencies:
+ dunder-proto: 1.0.1
+ es-object-atoms: 1.1.1
+
dependencies:
pump: 3.0.0
@@ -7738,16 +8037,40 @@ snapshots:
+ dependencies:
+ base64-js: 1.5.1
+ ecdsa-sig-formatter: 1.0.11
+ gaxios: 6.7.1([email protected])
+ gcp-metadata: 6.1.1([email protected])
+ gtoken: 7.1.0([email protected])
+ jws: 4.0.0
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
+
dependencies:
get-intrinsic: 1.2.4
+
dependencies:
array-keyed-map: 2.1.3
+ dependencies:
+ gaxios: 6.7.1([email protected])
+ jws: 4.0.0
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
@@ -7760,6 +8083,12 @@ snapshots:
+
+ dependencies:
+ has-symbols: 1.0.3
+
dependencies:
function-bind: 1.1.2
@@ -7776,6 +8105,8 @@ snapshots:
dependencies:
whatwg-encoding: 3.1.1
+
@@ -7801,6 +8132,14 @@ snapshots:
statuses: 2.0.1
toidentifier: 1.0.1
+ dependencies:
+ '@tootallnate/once': 2.0.0
+ agent-base: 6.0.2
+ debug: 4.3.5
+ transitivePeerDependencies:
+ - supports-color
+
dependencies:
agent-base: 7.1.1
@@ -8073,16 +8412,16 @@ snapshots:
- babel-plugin-macros
- supports-color
dependencies:
'@jest/test-result': 29.7.0
'@jest/types': 29.6.3
chalk: 4.1.2
exit: 0.1.2
import-local: 3.1.0
jest-util: 29.7.0
jest-validate: 29.7.0
yargs: 17.7.2
@@ -8092,7 +8431,7 @@ snapshots:
- supports-color
- ts-node
dependencies:
'@babel/core': 7.24.6
'@jest/test-sequencer': 29.7.0
@@ -8118,7 +8457,7 @@ snapshots:
strip-json-comments: 3.1.1
optionalDependencies:
'@types/node': 20.14.1
- ts-node: 10.9.2(@types/[email protected])([email protected])
+ ts-node: 10.9.2(@types/[email protected])([email protected])
transitivePeerDependencies:
- babel-plugin-macros
- supports-color
@@ -8345,12 +8684,12 @@ snapshots:
merge-stream: 2.0.0
supports-color: 8.1.1
dependencies:
'@jest/types': 29.6.3
import-local: 3.1.0
transitivePeerDependencies:
- '@types/node'
- babel-plugin-macros
@@ -8414,6 +8753,10 @@ snapshots:
+ dependencies:
+ bignumber.js: 9.2.0
+
@@ -8443,6 +8786,17 @@ snapshots:
readable-stream: 2.3.8
setimmediate: 1.0.5
+ dependencies:
+ buffer-equal-constant-time: 1.0.1
+ ecdsa-sig-formatter: 1.0.11
+ safe-buffer: 5.2.1
+
+ dependencies:
+ jwa: 2.0.0
+ safe-buffer: 5.2.1
+
@@ -8556,6 +8910,8 @@ snapshots:
+
dependencies:
charenc: 0.0.2
@@ -8589,6 +8945,8 @@ snapshots:
+
@@ -9036,7 +9394,7 @@ snapshots:
csv-parse: 5.5.6
gpt3-tokenizer: 1.1.5
openai: 3.3.0
- typescript: 5.8.2
+ typescript: 5.8.3
uuid: 9.0.1
zod: 3.24.2
transitivePeerDependencies:
@@ -9090,10 +9448,10 @@ snapshots:
- supports-color
- utf-8-validate
dependencies:
'@puppeteer/browsers': 2.2.3
- cosmiconfig: 9.0.0([email protected])
+ cosmiconfig: 9.0.0([email protected])
devtools-protocol: 0.0.1299070
puppeteer-core: 22.12.1
transitivePeerDependencies:
@@ -9234,6 +9592,17 @@ snapshots:
path-parse: 1.0.7
supports-preserve-symlinks-flag: 1.0.0
+ dependencies:
+ '@types/request': 2.48.12
+ extend: 3.0.2
+ teeny-request: 9.0.0([email protected])
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
+
dependencies:
glob: 10.4.2
@@ -9407,6 +9776,12 @@ snapshots:
+ dependencies:
+ stubs: 3.0.0
+
+
dependencies:
fast-fifo: 1.3.2
@@ -9461,6 +9836,8 @@ snapshots:
+
dependencies:
bin-links: 4.0.4
@@ -9541,6 +9918,17 @@ snapshots:
mkdirp: 3.0.1
yallist: 5.0.0
+ dependencies:
+ http-proxy-agent: 5.0.0
+ https-proxy-agent: 5.0.1
+ node-fetch: 2.7.0([email protected])
+ stream-events: 1.0.5
+ uuid: 9.0.1
+ transitivePeerDependencies:
+ - encoding
+ - supports-color
+
dependencies:
'@istanbuljs/schema': 0.1.3
@@ -9591,17 +9979,17 @@ snapshots:
dependencies:
bs-logger: 0.2.6
fast-json-stable-stringify: 2.1.0
jest-util: 29.7.0
json5: 2.2.3
lodash.memoize: 4.1.2
make-error: 1.3.6
semver: 7.6.2
- typescript: 5.4.5
+ typescript: 5.8.3
yargs-parser: 21.1.1
optionalDependencies:
'@babel/core': 7.24.6
@@ -9609,7 +9997,7 @@ snapshots:
'@jest/types': 29.6.3
babel-jest: 29.7.0(@babel/[email protected])
dependencies:
'@cspotcode/source-map-support': 0.8.1
'@tsconfig/node10': 1.0.11
@@ -9623,7 +10011,7 @@ snapshots:
create-require: 1.1.1
diff: 4.0.2
make-error: 1.3.6
- typescript: 5.4.5
+ typescript: 5.8.3
v8-compile-cache-lib: 3.0.1
yn: 3.1.1
@@ -9646,9 +10034,7 @@ snapshots:
media-typer: 0.3.0
mime-types: 2.1.35
-
dependencies:
@@ -9709,6 +10095,8 @@ snapshots:
+
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index ac41b6e83..5432687f8 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -1,10 +1,10 @@
-import { ExtractorOptions } from "./../../lib/entities";
import { supabase_service } from "../supabase";
import { FirecrawlJob } from "../../types";
import { posthog } from "../posthog";
import "dotenv/config";
import { logger } from "../../lib/logger";
import { configDotenv } from "dotenv";
+import { Storage } from "@google-cloud/storage";
configDotenv();
function cleanOfNull<T>(x: T): T {
@@ -21,6 +21,43 @@ function cleanOfNull<T>(x: T): T {
}
}
+
+async function saveJobToGCS(job: FirecrawlJob, bucketName: string): Promise<void> {
+ try {
+ const storage = new Storage();
+ const bucket = storage.bucket(bucketName);
+ const blob = bucket.file(`${job.job_id}.json`);
+ await blob.save(JSON.stringify(job.docs), {
+ contentType: "application/json",
+ });
+ await blob.setMetadata({
+ metadata: {
+ job_id: job.job_id ?? null,
+ success: job.success,
+ message: job.message ?? null,
+ num_docs: job.num_docs,
+ time_taken: job.time_taken,
+ team_id: (job.team_id === "preview" || job.team_id?.startsWith("preview_"))? null : job.team_id,
+ mode: job.mode,
+ url: job.url,
+ crawler_options: job.crawlerOptions,
+ page_options: job.scrapeOptions,
+ origin: job.origin,
+ num_tokens: job.num_tokens ?? null,
+ retry: !!job.retry,
+ crawl_id: job.crawl_id ?? null,
+ tokens_billed: job.tokens_billed ?? null,
+ },
+ })
+ } catch (error) {
+ logger.error(`Error saving job to GCS`, {
+ error,
+ scrapeId: job.job_id,
+ jobId: job.job_id,
+ });
+ }
+}
+
async function indexJob(job: FirecrawlJob): Promise<void> {
try {
if (job.mode !== "single_urls" && job.mode !== "scrape") {
@@ -108,6 +145,10 @@ export async function logJob(job: FirecrawlJob, force: boolean = false) {
indexJob(job);
}
+ if (process.env.GCS_BUCKET_NAME) {
+ await saveJobToGCS(job, process.env.GCS_BUCKET_NAME);
+ }
+
if (force) {
let i = 0,
done = false;