Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(core): apply object searchParams in HTTP request options (#3622)
Adarsh Tiwarid32946efbdc9
diff --git a/packages/utils/src/internals/url.ts b/packages/utils/src/internals/url.tsindex c13d674ea..a8b0fe6d4 100644--- a/packages/utils/src/internals/url.ts+++ b/packages/utils/src/internals/url.ts@@ -23,7 +23,7 @@ export function applySearchParams(url: URL, searchParams: SearchParams | undefinnewSearchParams = searchParams;} else {newSearchParams = new URLSearchParams();- for (const [key, value] of Object.entries(newSearchParams)) {+ for (const [key, value] of Object.entries(searchParams)) {if (value === undefined) {newSearchParams.delete(key);} else if (value === null) {diff --git a/test/core/base_http_client.test.ts b/test/core/base_http_client.test.tsnew file mode 100644index 000000000..8f72daabb--- /dev/null+++ b/test/core/base_http_client.test.ts@@ -0,0 +1,64 @@+import { processHttpRequestOptions } from '@crawlee/core';++describe('processHttpRequestOptions', () => {+ test('applies search parameters to the request URL', () => {+ const request = processHttpRequestOptions({+ url: 'https://example.com/path?previous=value',+ searchParams: {+ query: 'hello world',+ page: 2,+ empty: null,+ skipped: undefined,+ },+ });++ expect(request.url.toString()).toBe('https://example.com/path?query=hello+world&page=2&empty=');+ });++ test('throws when multiple body options are provided', () => {+ expect(() =>+ processHttpRequestOptions({+ url: 'https://example.com',+ body: 'body',+ json: { hello: 'world' },+ }),+ ).toThrow('At most one of `body`, `form` and `json` may be specified in sendRequest arguments');+ });++ test('serializes form body and sets default content type', () => {+ const request = processHttpRequestOptions({+ url: 'https://example.com',+ form: {+ hello: 'world',+ },+ });++ expect(request.body).toBe('hello=world');+ expect(request.headers).toEqual({ 'content-type': 'application/x-www-form-urlencoded' });+ });++ test('serializes JSON body and keeps user content type', () => {+ const request = processHttpRequestOptions({+ url: 'https://example.com',+ headers: {+ 'content-type': 'application/vnd.api+json',+ },+ json: {+ hello: 'world',+ },+ });++ expect(request.body).toBe('{"hello":"world"}');+ expect(request.headers).toEqual({ 'content-type': 'application/vnd.api+json' });+ });++ test('sets basic authorization header from username and password', () => {+ const request = processHttpRequestOptions({+ url: 'https://example.com',+ username: 'user',+ password: 'pass',+ });++ expect(request.headers).toEqual({ authorization: 'Basic dXNlcjpwYXNz' });+ });+});