Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(memory-storage): dont fail when storage already purged (#1737)
JJetmar86940273dbac
diff --git a/packages/memory-storage/src/cache-helpers.ts b/packages/memory-storage/src/cache-helpers.tsindex 823f08753..2860b7e6d 100644--- a/packages/memory-storage/src/cache-helpers.ts+++ b/packages/memory-storage/src/cache-helpers.ts@@ -45,7 +45,10 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entrhasSeenMetadataFile = true;// we have found the store metadata file, build out information based on it- const metadata = JSON.parse(await readFile(resolve(datasetDir, entry.name), 'utf8')) as storage.DatasetInfo;+ const fileContent = await readFile(resolve(datasetDir, entry.name), 'utf8');+ if (!fileContent) continue;++ const metadata = JSON.parse(fileContent) as storage.DatasetInfo;id = metadata.id;name = metadata.name;itemCount = metadata.itemCount;@@ -56,7 +59,10 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entrcontinue;}- const entryContent = JSON.parse(await readFile(resolve(datasetDir, entry.name), 'utf8')) as storage.Dictionary;+ const fileContent = await readFile(resolve(datasetDir, entry.name), 'utf8');+ if (!fileContent) continue;++ const entryContent = JSON.parse(fileContent) as storage.Dictionary;const entryName = entry.name.split('.')[0];entries.set(entryName, entryContent);@@ -131,7 +137,10 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorageif (entry.isFile()) {if (entry.name === '__metadata__.json') {// we have found the store metadata file, build out information based on it- const metadata = JSON.parse(await readFile(resolve(keyValueStoreDir, entry.name), 'utf8')) as storage.KeyValueStoreInfo;+ const fileContent = await readFile(resolve(keyValueStoreDir, entry.name), 'utf8');+ if (!fileContent) continue;++ const metadata = JSON.parse(fileContent) as storage.KeyValueStoreInfo;id = metadata.id;name = metadata.name;createdAt = new Date(metadata.createdAt);@@ -143,7 +152,10 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorageif (entry.name.includes('.__metadata__.')) {// This is an entry's metadata file, we can use it to create/extend the record- const metadata = JSON.parse(await readFile(resolve(keyValueStoreDir, entry.name), 'utf8')) as Omit<InternalKeyRecord, 'value'>;+ const fileContent = await readFile(resolve(keyValueStoreDir, entry.name), 'utf8');+ if (!fileContent) continue;++ const metadata = JSON.parse(fileContent) as Omit<InternalKeyRecord, 'value'>;const newRecord = {...internalRecords.get(metadata.key),@@ -270,7 +282,10 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryNswitch (entry.name) {case '__metadata__.json': {// we have found the store metadata file, build out information based on it- const metadata = JSON.parse(await readFile(resolve(requestQueueDir, entry.name), 'utf8')) as storage.RequestQueueInfo;+ const fileContent = await readFile(resolve(requestQueueDir, entry.name), 'utf8');+ if (!fileContent) continue;++ const metadata = JSON.parse(fileContent) as storage.RequestQueueInfo;id = metadata.id;name = metadata.name;@@ -283,7 +298,10 @@ export async function findRequestQueueByPossibleId(client: MemoryStorage, entryNbreak;}default: {- const request = JSON.parse(await readFile(resolve(requestQueueDir, entry.name), 'utf8')) as InternalRequest;+ const fileContent = await readFile(resolve(requestQueueDir, entry.name), 'utf8');+ if (!fileContent) continue;++ const request = JSON.parse(fileContent) as InternalRequest;entries.push(request);}}