Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: Lint
Jan Buchar146e02e2d203
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 0fd68a0f6..211169937 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -1,5 +1,6 @@import { writeFile } from 'node:fs/promises';import { dirname } from 'node:path';+import type {AddRequestsBatchedOptions,AddRequestsBatchedResult,@@ -24,8 +25,8 @@ import type {SessionPoolOptions,SkippedRequestCallback,Source,- StatisticState,StatisticsOptions,+ StatisticState,} from '@crawlee/core';import {AutoscaledPool,@@ -36,9 +37,9 @@ import {ContextPipelineInterruptedError,CriticalError,Dataset,+ enqueueLinks,EnqueueStrategy,EventType,- enqueueLinks,KeyValueStore,LogLevel,mergeCookies,@@ -54,6 +55,7 @@ import {RetryRequestError,Router,ServiceLocator,+ serviceLocator,SessionError,SessionPool,Statistics,diff --git a/packages/impit-client/src/index.ts b/packages/impit-client/src/index.tsindex c4ff22fba..0be7642b9 100644--- a/packages/impit-client/src/index.ts+++ b/packages/impit-client/src/index.ts@@ -1,9 +1,10 @@-import { LruCache } from '@apify/datastructures';import type { CustomFetchOptions } from '@crawlee/http-client';import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';import { Impit, type ImpitOptions } from 'impit';import type { CookieJar as ToughCookieJar } from 'tough-cookie';+import { LruCache } from '@apify/datastructures';+export const Browser = {'Chrome': 'chrome','Firefox': 'firefox',diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex e7c950f35..d039a30f9 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -2,7 +2,7 @@ import { readFile, rm } from 'node:fs/promises';import type { Server } from 'node:http';import http from 'node:http';import type { AddressInfo } from 'node:net';-import log from '@apify/log';+import type { EnqueueLinksOptions, ErrorHandler, RequestHandler, RequestOptions, Source } from '@crawlee/basic';import {BasicCrawler,@@ -32,6 +32,8 @@ import { z } from 'zod';import { startExpressAppPromise } from 'test/shared/_helper.js';+import log from '@apify/log';+describe('BasicCrawler', () => {let logLevel: number;const localStorageEmulator = new MemoryStorageEmulator();diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex c57a05d94..8b7870ac3 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -1,7 +1,5 @@import type { Server } from 'node:http';-import { ENV_VARS } from '@apify/consts';-import log from '@apify/log';import { BROWSER_POOL_EVENTS, OperatingSystemsName, PuppeteerPlugin } from '@crawlee/browser-pool';import { BLOCKED_STATUS_CODES } from '@crawlee/core';import type { PuppeteerGoToOptions } from '@crawlee/puppeteer';@@ -14,6 +12,9 @@ import puppeteer from 'puppeteer';import { runExampleComServer } from 'test/shared/_helper.js';import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator.js';+import { ENV_VARS } from '@apify/consts';+import log from '@apify/log';+import type { TestCrawlingContext } from './basic_browser_crawler.js';import { BrowserCrawlerTest } from './basic_browser_crawler.js';