Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(core): apply object searchParams in HTTP request options (#3622)

Adarsh Tiwarid32946efbdc9
diff --git a/packages/utils/src/internals/url.ts b/packages/utils/src/internals/url.ts
index c13d674ea..a8b0fe6d4 100644
--- a/packages/utils/src/internals/url.ts
+++ b/packages/utils/src/internals/url.ts
@@ -23,7 +23,7 @@ export function applySearchParams(url: URL, searchParams: SearchParams | undefin
newSearchParams = searchParams;
} else {
newSearchParams = new URLSearchParams();
- for (const [key, value] of Object.entries(newSearchParams)) {
+ for (const [key, value] of Object.entries(searchParams)) {
if (value === undefined) {
newSearchParams.delete(key);
} else if (value === null) {
diff --git a/test/core/base_http_client.test.ts b/test/core/base_http_client.test.ts
new file mode 100644
index 000000000..8f72daabb
--- /dev/null
+++ b/test/core/base_http_client.test.ts
@@ -0,0 +1,64 @@
+import { processHttpRequestOptions } from '@crawlee/core';
+
+describe('processHttpRequestOptions', () => {
+ test('applies search parameters to the request URL', () => {
+ const request = processHttpRequestOptions({
+ url: 'https://example.com/path?previous=value',
+ searchParams: {
+ query: 'hello world',
+ page: 2,
+ empty: null,
+ skipped: undefined,
+ },
+ });
+
+ expect(request.url.toString()).toBe('https://example.com/path?query=hello+world&page=2&empty=');
+ });
+
+ test('throws when multiple body options are provided', () => {
+ expect(() =>
+ processHttpRequestOptions({
+ url: 'https://example.com',
+ body: 'body',
+ json: { hello: 'world' },
+ }),
+ ).toThrow('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');
+ });
+
+ test('serializes form body and sets default content type', () => {
+ const request = processHttpRequestOptions({
+ url: 'https://example.com',
+ form: {
+ hello: 'world',
+ },
+ });
+
+ expect(request.body).toBe('hello=world');
+ expect(request.headers).toEqual({ 'content-type': 'application/x-www-form-urlencoded' });
+ });
+
+ test('serializes JSON body and keeps user content type', () => {
+ const request = processHttpRequestOptions({
+ url: 'https://example.com',
+ headers: {
+ 'content-type': 'application/vnd.api+json',
+ },
+ json: {
+ hello: 'world',
+ },
+ });
+
+ expect(request.body).toBe('{"hello":"world"}');
+ expect(request.headers).toEqual({ 'content-type': 'application/vnd.api+json' });
+ });
+
+ test('sets basic authorization header from username and password', () => {
+ const request = processHttpRequestOptions({
+ url: 'https://example.com',
+ username: 'user',
+ password: 'pass',
+ });
+
+ expect(request.headers).toEqual({ authorization: 'Basic dXNlcjpwYXNz' });
+ });
+});