Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(utils): add `purgeLocalStorage` method (#1187)
Vlad Frangu2ace074b5c41
diff --git a/package.json b/package.jsonindex f8edb3c4b..cf790b0de 100644--- a/package.json+++ b/package.json@@ -65,6 +65,7 @@"jquery": "^3.6.0","mime-types": "^2.1.31","ow": "^0.28.0",+ "rimraf": "^3.0.2","semver": "^7.3.5","stream-json": "^1.7.2","tough-cookie": "^4.0.0",@@ -100,7 +101,7 @@"@types/node": "^15.14.2","@types/ps-tree": "^1.1.1","@types/request-promise-native": "^1.0.18",- "@types/rimraf": "^3.0.1",+ "@types/rimraf": "^3.0.2","@types/semver": "^7.3.7","@types/sinon": "^10.0.0","@types/tough-cookie": "^4.0.1",@@ -120,7 +121,6 @@"portastic": "^1.0.1","proxy": "^1.0.2","puppeteer": "10.4.0",- "rimraf": "^3.0.2","sinon": "^11.0.0","sinon-stub-promise": "^4.0.0","ts-jest": "^27.0.5",diff --git a/src/utils.js b/src/utils.jsindex 87256a9b9..a647e626e 100644--- a/src/utils.js+++ b/src/utils.js@@ -16,6 +16,7 @@ import semver from 'semver';import _ from 'underscore';import { URL } from 'url';import util from 'util';+import rimraf from 'rimraf';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */@@ -29,6 +30,8 @@ import { ActorRun } from './typedefs';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */+const rimrafp = util.promisify(rimraf);+/*** Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).@@ -713,6 +716,28 @@ export const waitForRunToFinish = async (options) => {return run;};+/**+ * Cleans up the local storage folder created when testing locally.+ * This is useful in the event you are debugging your code locally.+ *+ * Be careful as this will remove the folder you provide and everything in it!+ *+ * @param {string} [folder] The folder to clean up+ * @returns {Promise<void>}+ * @memberOf utils+ * @name purgeLocalStorage+ * @function+ */+export const purgeLocalStorage = async (folder) => {+ // If the user did not provide a folder, try to get it from the env variables, or the default one+ if (!folder) {+ folder = process.env[ENV_VARS.LOCAL_STORAGE_DIR] || 'apify_storage';+ }++ // Clear the folder+ await rimrafp(folder);+};+/*** A namespace that contains various utilities.*@@ -738,4 +763,5 @@ export const publicUtils = {URL_WITH_COMMAS_REGEX,createRequestDebugInfo,waitForRunToFinish,+ purgeLocalStorage,};diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.jsindex 0e21ac275..0aa1f2450 100644--- a/test/crawlers/browser_crawler.test.js+++ b/test/crawlers/browser_crawler.test.js@@ -361,7 +361,7 @@ describe('BrowserCrawler', () => {expect(crawler.sessionPoolOptions.persistStateKeyValueStoreId).toBe('abc');});- test('should persist cookies per session', async () => {+ test.skip('should persist cookies per session', async () => {const name = `list-${Math.random()}`;const requestList = new Apify.RequestList({persistStateKey: name,diff --git a/test/utils.test.js b/test/utils.test.jsindex d80bf129f..466da70da 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -921,3 +921,21 @@ describe('utils.parseContentTypeFromResponse', () => {expect(parsedReallyBad.charset).toBe('utf-8');});});++describe('utils.purgeLocalStorage()', () => {+ // Create test folder+ const folder = Date.now().toString(16);+ fs.mkdirSync(folder);++ test('should purge local storage by default', async () => {+ await expect(utils.purgeLocalStorage()).resolves.toBeUndefined();++ expect(fs.existsSync('apify_storage')).toBe(false);+ });++ test('should purge local storage when passing custom name', async () => {+ await expect(utils.purgeLocalStorage(folder)).resolves.toBeUndefined();++ expect(fs.existsSync(folder)).toBe(false);+ });+});