Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Make new Request validation compatible with old request queues

Ondra Urbane463942d8a05
diff --git a/src/storages/request_queue.js b/src/storages/request_queue.js
index 735a0f62a..15c9d932f 100644
--- a/src/storages/request_queue.js
+++ b/src/storages/request_queue.js
@@ -237,9 +237,18 @@ export class RequestQueue {
// TODO: Could we also use requestsCache here? It would be consistent with addRequest()
// Downside is that it wouldn't reflect changes from outside...
- const obj = await this.client.getRequest(id);
+ const requestOptions = await this.client.getRequest(id);
+ if (!requestOptions) return null;
- return obj ? new Request(obj) : null;
+ // TODO: compatibility fix for old/broken request queues with null Request props
+ const optionsWithoutNulls = Object.entries(requestOptions).reduce((opts, [key, value]) => {
+ if (value !== null) {
+ opts[key] = value;
+ }
+ return opts;
+ }, {});
+
+ return new Request(optionsWithoutNulls);
}
/**
diff --git a/test/storages/request_queue.test.js b/test/storages/request_queue.test.js
index ef4810385..920b190bc 100644
--- a/test/storages/request_queue.test.js
+++ b/test/storages/request_queue.test.js
@@ -594,4 +594,28 @@ describe('RequestQueue remote', () => {
expect(deleteMock).toHaveBeenCalledTimes(1);
expect(deleteMock).toHaveBeenLastCalledWith();
});
+
+ test('getRequest should remove nulls from stored requests', async () => {
+ const url = 'http://example.com';
+ const method = 'POST';
+ const queue = new RequestQueue({ id: 'some-id', name: 'some-name', client: apifyClient });
+ const getRequestMock = jest
+ .spyOn(queue.client, 'getRequest')
+ .mockResolvedValueOnce({
+ url,
+ loadedUrl: null,
+ errorMessages: null,
+ handledAt: null,
+ method,
+ });
+
+ const request = await queue.getRequest('abc');
+ expect(getRequestMock).toHaveBeenCalledTimes(1);
+ expect(getRequestMock).toHaveBeenLastCalledWith('abc');
+ expect(request.url).toBe(url);
+ expect(request.loadedUrl).toBeUndefined();
+ expect(request.errorMessages).toEqual([]);
+ expect(request.handledAt).toBeUndefined();
+ expect(request.method).toBe(method);
+ });
});