Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: fix broken types in docs examples (#3287)
Jindřich Bär6361d064c413
diff --git a/docs/examples/file_download.ts b/docs/examples/file_download.tsindex a6b42555e..4ec682ea7 100644--- a/docs/examples/file_download.ts+++ b/docs/examples/file_download.ts@@ -2,11 +2,11 @@ import { FileDownload } from 'crawlee';// Create a FileDownload - a custom crawler instance that will download files from URLs.const crawler = new FileDownload({- async requestHandler({ body, request, contentType, getKeyValueStore }) {+ async requestHandler({ request, response, contentType, getKeyValueStore }) {const url = new URL(request.url);const kvs = await getKeyValueStore();- await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });+ await kvs.setValue(url.pathname.replace(/\//g, '_'), response.body, { contentType: contentType.type });},});diff --git a/docs/examples/file_download_stream.ts b/docs/examples/file_download_stream.tsindex a7f39a70f..8931ecc2c 100644--- a/docs/examples/file_download_stream.ts+++ b/docs/examples/file_download_stream.ts@@ -23,32 +23,27 @@ function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; t// Create a FileDownload - a custom crawler instance that will download files from URLs.const crawler = new FileDownload({- async streamHandler({ stream, request, log, getKeyValueStore }) {+ async requestHandler({ response, request, log, getKeyValueStore }) {const url = new URL(request.url);log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`);- await new Promise<void>((resolve, reject) => {- // With the 'response' event, we have received the headers of the response.- stream.on('response', async (response) => {- const kvs = await getKeyValueStore();- await kvs.setValue(- url.pathname.replace(/\//g, '_'),- pipeline(- stream,- createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }),- (error) => {- if (error) reject(error);- },- ),- { contentType: response.headers['content-type'] },- );-- log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);-- resolve();- });- });+ if (!response.body) return;++ const kvs = await getKeyValueStore();+ await kvs.setValue(+ url.pathname.replace(/\//g, '_'),+ pipeline(+ response.body,+ createProgressTracker({ url, log, totalBytes: Number(response.headers.get('content-length')) }),+ (error) => {+ if (error) log.error(`Failed to download ${url}: ${error.message}`);+ },+ ),+ response.headers.get('content-type') ? { contentType: response.headers.get('content-type')! } : {},+ );++ log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);},});diff --git a/docs/guides/custom-http-client/implementation.ts b/docs/guides/custom-http-client/implementation.tsindex 504f0b532..17684b5b6 100644--- a/docs/guides/custom-http-client/implementation.ts+++ b/docs/guides/custom-http-client/implementation.ts@@ -11,7 +11,7 @@ import { Readable } from 'node:stream';export class CustomHttpClient implements BaseHttpClient {async sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(request: HttpRequest<TResponseType>,- ): Promise<HttpResponse<TResponseType>> {+ ): Promise<Response> {const requestHeaders = new Headers();for (let [headerName, headerValues] of Object.entries(request.headers ?? {})) {if (headerValues === undefined) {@@ -27,96 +27,20 @@ export class CustomHttpClient implements BaseHttpClient {}}- const response = await fetch(request.url, {+ return fetch(request.url, {method: request.method,headers: requestHeaders,- body: request.body as string, // TODO implement stream/generator handling+ body: request.body as string,signal: request.signal,- // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)});-- const headers: Record<string, string> = {};-- response.headers.forEach((value, headerName) => {- headers[headerName] = value;- });-- return {- complete: true,- request,- url: response.url,- statusCode: response.status,- redirectUrls: [], // TODO you need to handle redirects manually to track them- headers,- trailers: {}, // TODO not supported by fetch- ip: undefined,- body:- request.responseType === 'text'- ? await response.text()- : request.responseType === 'json'- ? await response.json()- : Buffer.from(await response.text()),- };}- async stream(request: HttpRequest, _onRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {- const fetchResponse = await fetch(request.url, {+ async stream(request: HttpRequest, _onRedirect?: RedirectHandler): Promise<Response> {+ return fetch(request.url, {method: request.method,headers: new Headers(),- body: request.body as string, // TODO implement stream/generator handling+ body: request.body as string,signal: request.signal,- // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)});-- const headers: Record<string, string> = {}; // TODO same as in sendRequest()-- async function* read() {- const reader = fetchResponse.body?.getReader();-- const stream = new ReadableStream({- start(controller) {- if (!reader) {- return null;- }- return pump();- function pump(): Promise<void> {- return reader!.read().then(({ done, value }) => {- // When no more data needs to be consumed, close the stream- if (done) {- controller.close();- return;- }- // Enqueue the next data chunk into our target stream- controller.enqueue(value);- return pump();- });- }- },- });-- for await (const chunk of stream) {- yield chunk;- }- }-- const response = {- complete: false,- request,- url: fetchResponse.url,- statusCode: fetchResponse.status,- redirectUrls: [], // TODO you need to handle redirects manually to track them- headers,- trailers: {}, // TODO not supported by fetch- ip: undefined,- stream: Readable.from(read()),- get downloadProgress() {- return { percent: 0, transferred: 0 }; // TODO track this- },- get uploadProgress() {- return { percent: 0, transferred: 0 }; // TODO track this- },- };-- return response;}}diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdxindex 8bf385f1c..853042064 100644--- a/docs/guides/proxy_management.mdx+++ b/docs/guides/proxy_management.mdx@@ -31,7 +31,7 @@ import InspectionPuppeteerSource from '!!raw-loader!./proxy_management_inspectioand most effective ways of preventing access to a website. It is therefore paramount fora good web scraping library to provide easy to use but powerful tools which can work aroundIP blocking. The most powerful weapon in our anti IP blocking arsenal is a-[proxy server](https://en.wikipedia.org/wiki/Proxy_server).+[proxy server](https://en.wikipedia.org/wiki/Proxy_server).With Crawlee we can use our own proxy servers or proxy servers acquired fromthird-party providers.@@ -105,7 +105,7 @@ You can also provide a list of proxy tiers to the `ProxyConfiguration` class. Th:::warning-Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)).+Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)).Using this configuration through the `newUrl` calls will not yield the expected results.@@ -162,9 +162,7 @@ Our crawlers will now use the selected proxies for all connections.## IP Rotation and session management-​<ApiLink to="core/class/ProxyConfiguration#newUrl">`proxyConfiguration.newUrl()`</ApiLink> allows us to pass a `sessionId` parameter. It will then be used to create a `sessionId`-`proxyUrl` pair, and subsequent `newUrl()` calls with the same `sessionId` will always return the same `proxyUrl`. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and <ApiLink to="core/class/SessionPool">`SessionPool`</ApiLink> class for more information on how keeping a real session helps us avoid blocking.--When no `sessionId` is provided, our proxy URLs are rotated round-robin.+Each call to <ApiLink to="core/class/ProxyConfiguration#newUrl">`proxyConfiguration.newUrl()`</ApiLink> generates a new proxy URL. Crawler instances pair these URLs with `Session` instances and rotate those together with browser fingerprints, impersonated headers, and more. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and <ApiLink to="core/class/SessionPool">`SessionPool`</ApiLink> class for more information on how keeping a real session helps us avoid blocking.<Tabs groupId="proxy_session_management"><TabItem value="http" label="HttpCrawler">diff --git a/docs/guides/proxy_management_session_standalone.ts b/docs/guides/proxy_management_session_standalone.tsindex bc2010f79..dec095d03 100644--- a/docs/guides/proxy_management_session_standalone.ts+++ b/docs/guides/proxy_management_session_standalone.ts@@ -4,10 +4,4 @@ const proxyConfiguration = new ProxyConfiguration({/* opts */});-const sessionPool = await SessionPool.open({- /* opts */-});--const session = await sessionPool.getSession();--const proxyUrl = await proxyConfiguration.newUrl(session.id);+const proxyUrl = await proxyConfiguration.newUrl();diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.tsindex c7b7ec37c..38b65ee9d 100644--- a/docs/guides/session_management_basic.ts+++ b/docs/guides/session_management_basic.ts@@ -1,5 +1,6 @@import { BasicCrawler, ProxyConfiguration } from 'crawlee';-import { gotScraping } from 'got-scraping';+import { Impit } from 'impit';+import { Cookie } from 'tough-cookie';const proxyConfiguration = new ProxyConfiguration({/* opts */@@ -12,22 +13,19 @@ const crawler = new BasicCrawler({sessionPoolOptions: { maxPoolSize: 100 },async requestHandler({ request, session }) {const { url } = request;- const requestOptions = {- url,- // We use session id in order to have the same proxyUrl- // for all the requests using the same session.- proxyUrl: await proxyConfiguration.newUrl(session?.id),- throwHttpErrors: false,+ const client = new Impit({+ proxyUrl: await proxyConfiguration.newUrl(),+ ignoreTlsErrors: true,headers: {// If you want to use the cookieJar.// This way you get the Cookie headers string from session.- Cookie: session?.getCookieString(url),+ Cookie: session?.getCookieString(url) ?? '',},- };+ });let response;try {- response = await gotScraping(requestOptions);+ response = await client.fetch(url);} catch (e) {if (e === 'SomeNetworkError') {// If a network error happens, such as timeout, socket hangup, etc.@@ -39,9 +37,9 @@ const crawler = new BasicCrawler({}// Automatically retires the session based on response HTTP status code.- session?.retireOnBlockedStatusCodes(response.statusCode);+ session?.retireOnBlockedStatusCodes(response.status);- if (response.body.includes('You are blocked!')) {+ if ((await response.text()).includes('You are blocked!')) {// You are sure it is blocked.// This will throw away the session.session?.retire();@@ -51,6 +49,17 @@ const crawler = new BasicCrawler({// No need to call session.markGood -> BasicCrawler calls it for you.// If you want to use the CookieJar in session you need.- session?.setCookiesFromResponse(response);+ if (response.headers.has('set-cookie')) {+ const newCookies = response.headers+ .get('set-cookie')+ ?.split(';')+ .map((x) => Cookie.parse(x));++ newCookies?.forEach((cookie) => {+ if (cookie) {+ session?.cookieJar?.setCookie(cookie, url);+ }+ });+ }},});diff --git a/docs/package.json b/docs/package.jsonindex 51ae4027e..41a19eb45 100644--- a/docs/package.json+++ b/docs/package.json@@ -10,6 +10,7 @@"typescript": "^6.0.0"},"dependencies": {+ "impit": "^0.7.1","playwright-extra": "^4.3.6","puppeteer-extra": "^3.3.6","puppeteer-extra-plugin-stealth": "^2.11.2"