Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(scrapeURL): support cookies in safeFetch (#1688)
Gergő Móricz125e1ada45a6
diff --git a/apps/api/package.json b/apps/api/package.jsonindex f821c240e..7be4771c4 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -99,6 +99,7 @@"git-diff": "^2.0.6","glob": "^10.4.2","gpt3-tokenizer": "^1.1.5",+ "http-cookie-agent": "^7.0.1","ioredis": "^5.4.1","ip-address": "^10.0.1","joplin-turndown-plugin-gfm": "^1.0.12",@@ -132,10 +133,11 @@"supabase": "^1.77.9","systeminformation": "^5.22.11","tldts": "^6.1.75",+ "tough-cookie": "^5.1.2","turndown": "^7.1.3","turndown-plugin-gfm": "^1.0.2","typesense": "^1.5.4",- "undici": "^6.20.1",+ "undici": "^7.10.0","unstructured-client": "^0.11.3","uuid": "^10.0.0","winston": "^3.14.2",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 2bb103b60..8614d62d8 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -149,6 +149,9 @@ importers:gpt3-tokenizer:specifier: ^1.1.5version: 1.1.5+ http-cookie-agent:+ specifier: ^7.0.1+ version: 7.0.1([email protected])([email protected])ioredis:specifier: ^5.4.1version: 5.4.1@@ -248,6 +251,9 @@ importers:tldts:specifier: ^6.1.75version: 6.1.75+ tough-cookie:+ specifier: ^5.1.2+ version: 5.1.2turndown:specifier: ^7.1.3version: 7.2.0@@ -258,8 +264,8 @@ importers:specifier: ^1.5.4version: 1.8.2(@babel/[email protected])undici:- specifier: ^6.20.1- version: 6.20.1+ specifier: ^7.10.0+ version: 7.10.0unstructured-client:specifier: ^0.11.3version: 0.11.3([email protected])@@ -1921,10 +1927,6 @@ packages:resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}engines: {node: '>= 6.0.0'}- resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==}- engines: {node: '>= 14'}-resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==}engines: {node: '>= 14'}@@ -2981,6 +2983,16 @@ packages:resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}+ resolution: {integrity: sha512-lZHFZUdPTw64PdksQac5xbUd4NWjUbyDYnvR//2sbLpcC4UqEUW0x/6O+rDntVzJzJ07QvhtL5XZSC+c5EK+IQ==}+ engines: {node: '>=20.0.0'}+ peerDependencies:+ tough-cookie: ^4.0.0 || ^5.0.0+ undici: ^7.0.0+ peerDependenciesMeta:+ undici:+ optional: true+resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}engines: {node: '>= 0.8'}@@ -4523,8 +4535,8 @@ packages:resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}hasBin: true- resolution: {integrity: sha512-Ek7HndSVkp10hmHP9V4qZO1u+pn1RU5sI0Fw+jCU3lyvuMZcgqsNgc6CmJJZyByK4Vm/qotGRJlfgAX8q+4JiA==}+ resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==}engines: {node: '>=16'}@@ -4627,9 +4639,9 @@ packages:resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==}- resolution: {integrity: sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==}- engines: {node: '>=18.17'}+ resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==}+ engines: {node: '>=20.18.1'}resolution: {integrity: sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==}@@ -7213,12 +7225,6 @@ snapshots:transitivePeerDependencies:- supports-color- dependencies:- debug: 4.3.5- transitivePeerDependencies:- - supports-color-@@ -8373,6 +8379,13 @@ snapshots:domutils: 3.1.0entities: 4.5.0+ dependencies:+ agent-base: 7.1.3+ tough-cookie: 5.1.2+ optionalDependencies:+ undici: 7.10.0+dependencies:depd: 2.0.0@@ -8391,7 +8404,7 @@ snapshots:dependencies:- agent-base: 7.1.1+ agent-base: 7.1.3debug: 4.3.5transitivePeerDependencies:- supports-color@@ -8432,14 +8445,14 @@ snapshots:dependencies:- agent-base: 7.1.1+ agent-base: 7.1.3debug: 4.3.5transitivePeerDependencies:- supports-colordependencies:- agent-base: 7.1.1+ agent-base: 7.1.3debug: 4.3.5transitivePeerDependencies:- supports-color@@ -8987,7 +9000,7 @@ snapshots:rrweb-cssom: 0.8.0saxes: 6.0.0symbol-tree: 3.2.4- tough-cookie: 5.1.1+ tough-cookie: 5.1.2w3c-xmlserializer: 5.0.0webidl-conversions: 7.0.0whatwg-encoding: 3.1.1@@ -9462,7 +9475,7 @@ snapshots:dependencies:'@tootallnate/quickjs-emscripten': 0.23.0- agent-base: 7.1.1+ agent-base: 7.1.3debug: 4.3.5get-uri: 6.0.3http-proxy-agent: 7.0.2@@ -9665,7 +9678,7 @@ snapshots:dependencies:- agent-base: 7.1.1+ agent-base: 7.1.3debug: 4.3.5http-proxy-agent: 7.0.2https-proxy-agent: 7.0.5@@ -9995,7 +10008,7 @@ snapshots:dependencies:- agent-base: 7.1.1+ agent-base: 7.1.3debug: 4.3.5socks: 2.8.3transitivePeerDependencies:@@ -10222,7 +10235,7 @@ snapshots:dependencies:tldts: 6.1.75@@ -10314,7 +10327,7 @@ snapshots:- [email protected]: {}+ [email protected]: {}dependencies:diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex 1a959224e..d12ea4a75 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -3,6 +3,8 @@ import type { TLSSocket } from "tls";import * as undici from "undici";import { Address6 } from "ip-address";import { cacheableLookup } from "../../lib/cacheableLookup";+import { CookieJar } from "tough-cookie";+import { cookie } from "http-cookie-agent/undici";export class InsecureConnectionError extends Error {constructor() {@@ -53,7 +55,7 @@ export function makeSecureDispatcher(...options,};- const agent = process.env.PROXY_SERVER+ const baseAgent = process.env.PROXY_SERVER? new undici.ProxyAgent({uri: process.env.PROXY_SERVER.includes("://") ? process.env.PROXY_SERVER : ("http://" + process.env.PROXY_SERVER),token: process.env.PROXY_USERNAME@@ -63,6 +65,11 @@ export function makeSecureDispatcher(}): new undici.Agent(agentOpts);+ const cookieJar = new CookieJar();++ const agent = baseAgent+ .compose(cookie({ jar: cookieJar }));+agent.on("connect", (_, targets) => {const client: undici.Client = targets.slice(-1)[0] as undici.Client;const socketSymbol = Object.getOwnPropertySymbols(client).find(