Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Update openRequestList
Ondra Urbanc799dc547f3f
diff --git a/src/request_list.js b/src/request_list.jsindex 779902f32..31dc7047d 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -622,7 +622,8 @@ export const openRequestList = async (listName, sources, options = {}) => {const rl = new RequestList({...options,- stateKeyPrefix: listName,+ persistStateKey: listName ? `${listName}-${STATE_PERSISTENCE_KEY}` : null,+ persistSourcesKey: listName ? `${listName}-${SOURCES_PERSISTENCE_KEY}` : null,sources,});await rl.initialize();