Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): skip malformed cookie fragments in mergeCookies (#3822)
Anas Khanb3c3ef2cab17
diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.tsindex a97477ce7..7e8f4e12e 100644--- a/packages/core/src/cookie_utils.ts+++ b/packages/core/src/cookie_utils.ts@@ -116,7 +116,8 @@ export function mergeCookies(url: string, sourceCookies: string[]): string {// ignore extra spacesif (!cookieString) continue;- const cookie = Cookie.parse(cookieString)!;+ const cookie = Cookie.parse(cookieString);+ if (!cookie) throw new CookieParseError(cookieString);const similarKeyCookie = jar.getCookiesSync(url).find((c) => {return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();});diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex cfba7f7b8..5696dbc3e 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1169,6 +1169,15 @@ describe('CheerioCrawler', () => {);});+ test('mergeCookies() throws a contextual error for malformed cookie fragments', () => {+ expect(() => mergeCookies('https://example.com', ['valid=1; brokenfragment'])).toThrow(+ 'Could not parse cookie header string: brokenfragment',+ );+ expect(() => mergeCookies('https://example.com', ['sessionid'])).toThrow(+ 'Could not parse cookie header string: sessionid',+ );+ });+test('should use sessionId in proxyUrl when the session pool is enabled', async () => {const sourcesNew = [{ url: 'http://example.com/?q=1' }];const requestListNew = await RequestList.open({ sources: sourcesNew });