Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: improve file saving on memory storage (#1421)
Vlad Frangu75dc4bd633a8
diff --git a/packages/memory-storage/src/memory-storage.ts b/packages/memory-storage/src/memory-storage.tsindex 0230a6df2..80d301003 100644--- a/packages/memory-storage/src/memory-storage.ts+++ b/packages/memory-storage/src/memory-storage.ts@@ -2,16 +2,16 @@import type * as storage from '@crawlee/types';import type { Dictionary } from '@crawlee/types';import { s } from '@sapphire/shapeshift';-import { pathExistsSync } from 'fs-extra';-import { readdir, rm } from 'node:fs/promises';-import { join, resolve } from 'node:path';+import { ensureDir, pathExistsSync } from 'fs-extra';+import { rm, rename, readdir } from 'node:fs/promises';+import { resolve } from 'node:path';import { DatasetClient } from './resource-clients/dataset';import { DatasetCollectionClient } from './resource-clients/dataset-collection';import { KeyValueStoreClient } from './resource-clients/key-value-store';import { KeyValueStoreCollectionClient } from './resource-clients/key-value-store-collection';import { RequestQueueClient } from './resource-clients/request-queue';import { RequestQueueCollectionClient } from './resource-clients/request-queue-collection';-import { initWorkerIfNeeded } from './workers/instance';+import { initWorkerIfNeeded, promiseMap } from './workers/instance';export interface MemoryStorageOptions {/**@@ -114,32 +114,136 @@ export class MemoryStorage implements storage.StorageClient {* - local directory containing the default request queue.*/async purge(): Promise<void> {- const defaultDatasetPath = resolve(this.datasetsDirectory, 'default');- await this.removeFiles(defaultDatasetPath);+ // Datasets+ const datasets = await readdir(this.datasetsDirectory).catch(() => []);+ const datasetPromises: Promise<void>[] = [];- const defaultKeyValueStorePath = resolve(this.keyValueStoresDirectory, 'default');- await this.removeFiles(defaultKeyValueStorePath);+ for (const datasetFolder of datasets) {+ if (datasetFolder === 'default' || datasetFolder.startsWith('__CRAWLEE_TEMPORARY')) {+ datasetPromises.push((await this.batchRemoveFiles(resolve(this.datasetsDirectory, datasetFolder)))());+ }+ }++ void Promise.allSettled(datasetPromises);++ // Request queues+ const requestQueues = await readdir(this.requestQueuesDirectory).catch(() => []);+ const requestQueuePromises: Promise<void>[] = [];++ for (const requestQueueFolder of requestQueues) {+ if (requestQueueFolder === 'default' || requestQueueFolder.startsWith('__CRAWLEE_TEMPORARY')) {+ requestQueuePromises.push((await this.batchRemoveFiles(resolve(this.requestQueuesDirectory, requestQueueFolder)))());+ }+ }++ void Promise.allSettled(requestQueuePromises);++ // Key-value stores+ const keyValueStores = await readdir(this.keyValueStoresDirectory).catch(() => []);+ const keyValueStorePromises: Promise<void>[] = [];- const defaultRequestQueuePath = resolve(this.requestQueuesDirectory, 'default');- await this.removeFiles(defaultRequestQueuePath);+ for (const keyValueStoreFolder of keyValueStores) {+ if (keyValueStoreFolder.startsWith('__CRAWLEE_TEMPORARY')) {+ keyValueStorePromises.push((await this.batchRemoveFiles(resolve(this.keyValueStoresDirectory, keyValueStoreFolder)))());+ } else if (keyValueStoreFolder === 'default') {+ keyValueStorePromises.push((await this.handleDefaultKeyValueStore(resolve(this.keyValueStoresDirectory, keyValueStoreFolder)))());+ }+ }++ void Promise.allSettled(keyValueStorePromises);}- private async removeFiles(folder: string): Promise<void> {+ /**+ * This method should be called at the end of the process, to ensure all data is saved.+ */+ async teardown(): Promise<void> {+ const promises = [...promiseMap.values()].map(({ promise }) => promise);++ await Promise.all(promises);+ }++ private async handleDefaultKeyValueStore(folder: string): Promise<() => Promise<void>> {const storagePathExists = pathExistsSync(folder);+ const temporaryPath = resolve(folder, '../__CRAWLEE_MIGRATING_KEY_VALUE_STORE__');++ // For optimization, we want to only attempt to copy a few files from the default key-value store+ const possibleInputKeys = [+ 'INPUT',+ 'INPUT.json',+ 'INPUT.bin',+ 'INPUT.txt',+ ];if (storagePathExists) {- const direntNames = await readdir(folder);- const deletePromises = [];+ // Create temporary folder to save important files in+ await ensureDir(temporaryPath);++ // Go through each file and save the ones that are important+ for (const entity of possibleInputKeys) {+ const originalFilePath = resolve(folder, entity);+ const tempFilePath = resolve(temporaryPath, entity);++ try {+ await rename(originalFilePath, tempFilePath);+ } catch {+ // Ignore+ }+ }- for (const direntName of direntNames) {- const fileName = join(folder, direntName);+ // Remove the original folder and all its content+ const tempPathForOldFolder = resolve(folder, '../__OLD_DEFAULT__');+ await rename(folder, tempPathForOldFolder);- if (!fileName.match(/INPUT/)) {- deletePromises.push(rm(fileName, { recursive: true, force: true }));- }+ // Replace the temporary folder with the original folder+ await rename(temporaryPath, folder);++ // Remove the old folder+ return async () => (await this.batchRemoveFiles(tempPathForOldFolder))();+ }++ return () => Promise.resolve();+ }++ private async batchRemoveFiles(folder: string, counter = 0): Promise<() => Promise<void>> {+ const folderExists = pathExistsSync(folder);++ if (folderExists) {+ const temporaryFolder = folder.startsWith('__CRAWLEE_TEMPORARY_') ? folder : resolve(folder, `../__CRAWLEE_TEMPORARY_${counter}__`);++ try {+ // Rename the old folder to the new one to allow background deletions+ await rename(folder, temporaryFolder);+ } catch {+ // Folder exists already, try again with an incremented counter+ return this.batchRemoveFiles(folder, ++counter);}- await Promise.all(deletePromises);+ return async () => {+ // Read all files in the folder+ const entries = await readdir(temporaryFolder);++ let processed = 0;+ let promises: Promise<void>[] = [];++ for (const entry of entries) {+ processed++;+ promises.push(rm(resolve(temporaryFolder, entry), { force: true }));++ // Every 2000 files, delete them+ if (processed % 2000 === 0) {+ await Promise.allSettled(promises);+ promises = [];+ }+ }++ // Ensure last promises are handled+ await Promise.allSettled(promises);++ // Delete the folder itself+ await rm(temporaryFolder, { force: true, recursive: true });+ };}++ return () => Promise.resolve();}}diff --git a/packages/memory-storage/src/workers/file-storage-worker-emulator.ts b/packages/memory-storage/src/workers/file-storage-worker-emulator.tsindex cb2ec61c2..ba4a03422 100644--- a/packages/memory-storage/src/workers/file-storage-worker-emulator.ts+++ b/packages/memory-storage/src/workers/file-storage-worker-emulator.ts@@ -1,8 +1,12 @@+import { promiseMap } from './instance';import { handleMessage } from './worker-utils';export class FileStorageWorkerEmulator {- postMessage(value: any): void {- void handleMessage(value);+ async postMessage(value: any): Promise<void> {+ await handleMessage(value);++ promiseMap.get(value.messageId)?.resolve();+ promiseMap.delete(value.messageId);}terminate(): void {}diff --git a/packages/memory-storage/src/workers/file-storage-worker.ts b/packages/memory-storage/src/workers/file-storage-worker.tsindex f97719b73..7bb186838 100644--- a/packages/memory-storage/src/workers/file-storage-worker.ts+++ b/packages/memory-storage/src/workers/file-storage-worker.ts@@ -11,6 +11,6 @@ setInterval(() => {parentPort!.postMessage('ping');}, 30_000);-parentPort!.on('message', async (message: WorkerReceivedMessage) => {+parentPort!.on('message', async (message: WorkerReceivedMessage & { messageId: string }) => {await handleMessage(message);});diff --git a/packages/memory-storage/src/workers/instance.ts b/packages/memory-storage/src/workers/instance.tsindex f26544c88..55bad97a4 100644--- a/packages/memory-storage/src/workers/instance.ts+++ b/packages/memory-storage/src/workers/instance.ts@@ -1,13 +1,35 @@+import { randomUUID } from 'node:crypto';import { resolve } from 'node:path';import { Worker } from 'node:worker_threads';import type { WorkerReceivedMessage } from '../utils';import { memoryStorageLog } from '../utils';import { FileStorageWorkerEmulator } from './file-storage-worker-emulator';+// eslint-disable-next-line import/no-mutable-exportslet workerInstance: Worker | FileStorageWorkerEmulator;+export const promiseMap: Map<string, {+ promise: Promise<void>;+ resolve: () => void;+}> = new Map();+export function sendWorkerMessage(message: WorkerReceivedMessage) {- workerInstance.postMessage(message);+ const id = randomUUID();++ let promiseResolve: () => void;+ const promise = new Promise<void>((res) => {+ promiseResolve = res;+ });++ promiseMap.set(id, {+ promise,+ resolve: promiseResolve!,+ });++ void workerInstance.postMessage({+ ...message,+ messageId: id,+ });}export function initWorkerIfNeeded() {@@ -32,6 +54,15 @@ export function initWorkerIfNeeded() {memoryStorageLog.debug(`File storage worker exited with code ${code}`);initWorkerIfNeeded();});++ (workerInstance as Worker).on('message', (message: { messageId: string; type: 'ack' }) => {+ if (message.type !== 'ack') {+ return;+ }++ promiseMap.get(message.messageId)?.resolve();+ promiseMap.delete(message.messageId);+ });} else {workerInstance = new FileStorageWorkerEmulator();}diff --git a/packages/memory-storage/src/workers/worker-utils.ts b/packages/memory-storage/src/workers/worker-utils.tsindex a8e995617..d4fc122b3 100644--- a/packages/memory-storage/src/workers/worker-utils.ts+++ b/packages/memory-storage/src/workers/worker-utils.ts@@ -2,11 +2,12 @@ import log from '@apify/log';import { ensureDir } from 'fs-extra';import { rm, writeFile } from 'node:fs/promises';import { resolve } from 'node:path';+import { parentPort } from 'node:worker_threads';import type { WorkerDeleteEntryMessage, WorkerReceivedMessage, WorkerUpdateEntriesMessage, WorkerUpdateMetadataMessage } from '../utils';const workerLog = log.child({ prefix: 'MemoryStorageWorker' });-export async function handleMessage(message: WorkerReceivedMessage) {+export async function handleMessage(message: WorkerReceivedMessage & { messageId: string }) {switch (message.action) {case 'update-metadata':await updateMetadata(message);@@ -22,6 +23,11 @@ export async function handleMessage(message: WorkerReceivedMessage) {// we should be aware of themworkerLog.warning(`Unknown worker message action ${(message as WorkerReceivedMessage).action}`);}++ parentPort?.postMessage({+ type: 'ack',+ messageId: message.messageId,+ });}async function updateMetadata(message: WorkerUpdateMetadataMessage) {diff --git a/packages/types/src/storages.ts b/packages/types/src/storages.tsindex 8a0f08aa7..67e505115 100644--- a/packages/types/src/storages.ts+++ b/packages/types/src/storages.ts@@ -297,5 +297,6 @@ export interface StorageClient {requestQueues(): RequestQueueCollectionClient;requestQueue(id: string, options?: RequestQueueOptions): RequestQueueClient;purge?(): Promise<void>;+ teardown?(): Promise<void>;stats?: { rateLimitErrors: number[] };}