Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: support `DELETE` requests in `HttpCrawler` (#2039)

Hamza Alwan7ea5c4185b16
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 9f37a035d..2451506f0 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -814,6 +814,11 @@ export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any, H
}
});
+ // We need to end the stream for DELETE requests, otherwise it will hang.
+ if (options.method && ['DELETE', 'delete'].includes(options.method)) {
+ stream.end();
+ }
+
stream.on('error', reject);
stream.on('response', () => {
resolve(addResponsePropertiesToStream(stream));
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 0a83bc553..fde284516 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -507,7 +507,7 @@ describe('CheerioCrawler', () => {
});
});
- test('should ignore non http error status codes set by user', async () => {
+ test('should ignore http error status codes set by user', async () => {
const requestList = await getRequestListForMock({
headers: {
'content-type': 'text/plain',
@@ -534,7 +534,7 @@ describe('CheerioCrawler', () => {
expect(failed).toHaveLength(0);
});
- test('should throw and error on http error status codes set by user', async () => {
+ test('should throw an error on http error status codes set by user', async () => {
const requestList = await getRequestListForMirror();
const failed: Request[] = [];
@@ -1290,6 +1290,34 @@ describe('CheerioCrawler', () => {
expect(cheerioCrawler.requestHandler).toBeUndefined();
});
});
+
+ test('should work with delete requests', async () => {
+ const sources: Source[] = [1, 2, 3, 4].map((num) => {
+ return {
+ url: `${serverAddress}/special/mock?a=${num}`,
+ method: 'DELETE',
+ };
+ });
+ const requestList = await RequestList.open(null, sources);
+
+ const failed: Request[] = [];
+
+ const cheerioCrawler = new CheerioCrawler({
+ requestList,
+ maxConcurrency: 1,
+ maxRequestRetries: 0,
+ navigationTimeoutSecs: 5,
+ requestHandlerTimeoutSecs: 5,
+ requestHandler: async () => {},
+ failedRequestHandler: async ({ request }) => {
+ failed.push(request);
+ },
+ });
+
+ await cheerioCrawler.run();
+
+ expect(failed).toHaveLength(0);
+ });
});
async function getRequestListForMock(mockData: Dictionary, pathName = 'special/mock') {
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index b6e2671d1..c95f63e8a 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -279,7 +279,7 @@ test('POST with undefined (empty) payload', async () => {
expect(results).toStrictEqual(['']);
});
-test('should ignore non http error status codes set by user', async () => {
+test('should ignore http error status codes set by user', async () => {
const failed: any[] = [];
const crawler = new HttpCrawler({
@@ -298,7 +298,7 @@ test('should ignore non http error status codes set by user', async () => {
expect(failed).toHaveLength(0);
});
-test('should throw and error on http error status codes set by user', async () => {
+test('should throw an error on http error status codes set by user', async () => {
const failed: any[] = [];
const crawler = new HttpCrawler({
@@ -316,3 +316,25 @@ test('should throw and error on http error status codes set by user', async () =
expect(crawler.autoscaledPool.minConcurrency).toBe(2);
expect(failed).toHaveLength(1);
});
+
+test('should work with delete requests', async () => {
+ const failed: any[] = [];
+
+ const cheerioCrawler = new HttpCrawler({
+ maxConcurrency: 1,
+ maxRequestRetries: 0,
+ navigationTimeoutSecs: 5,
+ requestHandlerTimeoutSecs: 5,
+ requestHandler: async () => {},
+ failedRequestHandler: async ({ request }) => {
+ failed.push(request);
+ },
+ });
+
+ await cheerioCrawler.run([{
+ url: `${url}`,
+ method: 'DELETE',
+ }]);
+
+ expect(failed).toHaveLength(0);
+});