Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: handle extra cases for diffCookie (#1217)

Vlad Frangu32776bc813d6
diff --git a/src/crawlers/crawler_utils.js b/src/crawlers/crawler_utils.js
index 5f1fa5d6d..3e946495d 100644
--- a/src/crawlers/crawler_utils.js
+++ b/src/crawlers/crawler_utils.js
@@ -67,18 +67,22 @@ export function mergeCookies(url, sourceCookies) {
/**
* @param {string} url
- * @param {string} cookieString1
- * @param {string} cookieString2
+ * @param {string} [cookieString1='']
+ * @param {string} [cookieString2='']
* @return {string}
* @private
*/
-export function diffCookies(url, cookieString1, cookieString2) {
- if (cookieString1 === cookieString2) {
+export function diffCookies(url, cookieString1 = '', cookieString2 = '') {
+ if (cookieString1 === cookieString2 || !cookieString2) {
return '';
}
- const cookies1 = cookieString1.split(/ *; */).map((cookie) => Cookie.parse(cookie));
- const cookies2 = cookieString2.split(/ *; */).map((cookie) => Cookie.parse(cookie));
+ if (!cookieString1) {
+ return cookieString2;
+ }
+
+ const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => Cookie.parse(cookie));
+ const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => Cookie.parse(cookie));
const added = cookies2.filter((newCookie) => {
return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
diff --git a/test/crawlers/crawler_utils.test.js b/test/crawlers/crawler_utils.test.js
new file mode 100644
index 000000000..c0346889d
--- /dev/null
+++ b/test/crawlers/crawler_utils.test.js
@@ -0,0 +1,28 @@
+import { diffCookies } from '../../build/crawlers/crawler_utils';
+
+const cookie1 = 'test_cookie=1';
+const cookie2 = 'new_test_cookie=2';
+
+const mergedCookies = [cookie1, cookie2].join(';');
+
+const url = 'https://example.com';
+
+describe('CrawlerUtils', () => {
+ describe('diffCookies', () => {
+ test('when no cookies are provided for cookiesString2, return an empty string', () => {
+ expect(diffCookies(url, cookie1, '')).toEqual('');
+ });
+
+ test('when no cookies are provided for cookiesString1, return cookiesString2', () => {
+ expect(diffCookies(url, '', cookie1)).toEqual(cookie1);
+ });
+
+ test('when new cookies are provided for cookiesString2, return them', () => {
+ expect(diffCookies(url, cookie1, cookie2)).toEqual(cookie2);
+ });
+
+ test('when new cookies include previous cookies for cookiesString2, return only the new ones', () => {
+ expect(diffCookies(url, cookie1, mergedCookies)).toEqual(cookie2);
+ });
+ });
+});