Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: fix PLATFORM e2e tests (#1558)
Szymon Marczak73e105a2654a
diff --git a/package-lock.json b/package-lock.jsonindex 448e81d38..82bb1c420 100644--- a/package-lock.json+++ b/package-lock.json@@ -46,6 +46,7 @@"fs-extra": "^10.1.0","gen-esm-wrapper": "^1.1.3","globby": "^13.1.2",+ "got": "^12.5.0","husky": "^8.0.1","jest": "^29.1.2","lerna": "^5.5.4",@@ -8652,6 +8653,31 @@"url": "https://github.com/sponsors/sindresorhus"}},+ "node_modules/got": {+ "version": "12.5.0",+ "resolved": "https://registry.npmjs.org/got/-/got-12.5.0.tgz",+ "integrity": "sha512-/Bneo/L6bLN1wDyJCeRZ3CLoixvwb9v3rE3IHulFSfTHwP85xSr4QatA8K0c6GlL5+mc4IZ57BzluNZJiXvHIg==",+ "dev": true,+ "dependencies": {+ "@sindresorhus/is": "^5.2.0",+ "@szmarczak/http-timer": "^5.0.1",+ "cacheable-lookup": "^6.0.4",+ "cacheable-request": "^10.1.2",+ "decompress-response": "^6.0.0",+ "form-data-encoder": "^2.1.2",+ "get-stream": "^6.0.1",+ "http2-wrapper": "^2.1.10",+ "lowercase-keys": "^3.0.0",+ "p-cancelable": "^3.0.0",+ "responselike": "^3.0.0"+ },+ "engines": {+ "node": ">=14.16"+ },+ "funding": {+ "url": "https://github.com/sindresorhus/got?sponsor=1"+ }+ },"node_modules/got-cjs": {"version": "12.3.1","resolved": "https://registry.npmjs.org/got-cjs/-/got-cjs-12.3.1.tgz",@@ -8695,6 +8721,116 @@"node": ">=15.10.0"}},+ "node_modules/got/node_modules/@sindresorhus/is": {+ "version": "5.3.0",+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz",+ "integrity": "sha512-CX6t4SYQ37lzxicAqsBtxA3OseeoVrh9cSJ5PFYam0GksYlupRfy1A+Q4aYD3zvcfECLc0zO2u+ZnR2UYKvCrw==",+ "dev": true,+ "engines": {+ "node": ">=14.16"+ },+ "funding": {+ "url": "https://github.com/sindresorhus/is?sponsor=1"+ }+ },+ "node_modules/got/node_modules/@szmarczak/http-timer": {+ "version": "5.0.1",+ "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",+ "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",+ "dev": true,+ "dependencies": {+ "defer-to-connect": "^2.0.1"+ },+ "engines": {+ "node": ">=14.16"+ }+ },+ "node_modules/got/node_modules/cacheable-request": {+ "version": "10.2.0",+ "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.0.tgz",+ "integrity": "sha512-PLOMmzyOU3L5HECdZC0PTNawPdP/taOhCNrYUmZchrw7gwERMctfx2wZcndb63w5W06PTHC1Poq89J9chMZPLw==",+ "dev": true,+ "dependencies": {+ "get-stream": "^6.0.1",+ "http-cache-semantics": "^4.1.0",+ "keyv": "^4.5.0",+ "mimic-response": "^4.0.0",+ "normalize-url": "^7.1.0",+ "responselike": "^3.0.0"+ },+ "engines": {+ "node": ">=14.16"+ }+ },+ "node_modules/got/node_modules/form-data-encoder": {+ "version": "2.1.2",+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.2.tgz",+ "integrity": "sha512-FCaIOVTRA9E0siY6FeXid7D5yrCqpsErplUkE2a1BEiKj1BE9z6FbKB4ntDTwC4NVLie9p+4E9nX4mWwEOT05A==",+ "dev": true,+ "engines": {+ "node": ">= 14.17"+ }+ },+ "node_modules/got/node_modules/lowercase-keys": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",+ "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",+ "dev": true,+ "engines": {+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },+ "node_modules/got/node_modules/mimic-response": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",+ "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",+ "dev": true,+ "engines": {+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },+ "node_modules/got/node_modules/normalize-url": {+ "version": "7.1.0",+ "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.1.0.tgz",+ "integrity": "sha512-JgkdydFdLe1E5Q7DpLvKVyBZOOwXYGhIbMbOMm3lJ06XKzaiit+qo1HciO3z3IFklStfarzJHVQf9ZcNPTvZlw==",+ "dev": true,+ "engines": {+ "node": ">=12.20"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },+ "node_modules/got/node_modules/p-cancelable": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",+ "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==",+ "dev": true,+ "engines": {+ "node": ">=12.20"+ }+ },+ "node_modules/got/node_modules/responselike": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",+ "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",+ "dev": true,+ "dependencies": {+ "lowercase-keys": "^3.0.0"+ },+ "engines": {+ "node": ">=14.16"+ },+ "funding": {+ "url": "https://github.com/sponsors/sindresorhus"+ }+ },"node_modules/graceful-fs": {"version": "4.2.10","resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",@@ -24100,6 +24236,95 @@"slash": "^4.0.0"}},+ "got": {+ "version": "12.5.0",+ "resolved": "https://registry.npmjs.org/got/-/got-12.5.0.tgz",+ "integrity": "sha512-/Bneo/L6bLN1wDyJCeRZ3CLoixvwb9v3rE3IHulFSfTHwP85xSr4QatA8K0c6GlL5+mc4IZ57BzluNZJiXvHIg==",+ "dev": true,+ "requires": {+ "@sindresorhus/is": "^5.2.0",+ "@szmarczak/http-timer": "^5.0.1",+ "cacheable-lookup": "^6.0.4",+ "cacheable-request": "^10.1.2",+ "decompress-response": "^6.0.0",+ "form-data-encoder": "^2.1.2",+ "get-stream": "^6.0.1",+ "http2-wrapper": "^2.1.10",+ "lowercase-keys": "^3.0.0",+ "p-cancelable": "^3.0.0",+ "responselike": "^3.0.0"+ },+ "dependencies": {+ "@sindresorhus/is": {+ "version": "5.3.0",+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz",+ "integrity": "sha512-CX6t4SYQ37lzxicAqsBtxA3OseeoVrh9cSJ5PFYam0GksYlupRfy1A+Q4aYD3zvcfECLc0zO2u+ZnR2UYKvCrw==",+ "dev": true+ },+ "@szmarczak/http-timer": {+ "version": "5.0.1",+ "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",+ "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",+ "dev": true,+ "requires": {+ "defer-to-connect": "^2.0.1"+ }+ },+ "cacheable-request": {+ "version": "10.2.0",+ "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.0.tgz",+ "integrity": "sha512-PLOMmzyOU3L5HECdZC0PTNawPdP/taOhCNrYUmZchrw7gwERMctfx2wZcndb63w5W06PTHC1Poq89J9chMZPLw==",+ "dev": true,+ "requires": {+ "get-stream": "^6.0.1",+ "http-cache-semantics": "^4.1.0",+ "keyv": "^4.5.0",+ "mimic-response": "^4.0.0",+ "normalize-url": "^7.1.0",+ "responselike": "^3.0.0"+ }+ },+ "form-data-encoder": {+ "version": "2.1.2",+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.2.tgz",+ "integrity": "sha512-FCaIOVTRA9E0siY6FeXid7D5yrCqpsErplUkE2a1BEiKj1BE9z6FbKB4ntDTwC4NVLie9p+4E9nX4mWwEOT05A==",+ "dev": true+ },+ "lowercase-keys": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",+ "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",+ "dev": true+ },+ "mimic-response": {+ "version": "4.0.0",+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",+ "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",+ "dev": true+ },+ "normalize-url": {+ "version": "7.1.0",+ "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.1.0.tgz",+ "integrity": "sha512-JgkdydFdLe1E5Q7DpLvKVyBZOOwXYGhIbMbOMm3lJ06XKzaiit+qo1HciO3z3IFklStfarzJHVQf9ZcNPTvZlw==",+ "dev": true+ },+ "p-cancelable": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",+ "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==",+ "dev": true+ },+ "responselike": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",+ "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",+ "dev": true,+ "requires": {+ "lowercase-keys": "^3.0.0"+ }+ }+ }+ },"got-cjs": {"version": "12.3.1","resolved": "https://registry.npmjs.org/got-cjs/-/got-cjs-12.3.1.tgz",diff --git a/package.json b/package.jsonindex 19a9f1091..ea9566231 100644--- a/package.json+++ b/package.json@@ -84,6 +84,7 @@"fs-extra": "^10.1.0","gen-esm-wrapper": "^1.1.3","globby": "^13.1.2",+ "got": "^12.5.0","husky": "^8.0.1","jest": "^29.1.2","lerna": "^5.5.4",diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex af891e760..539a4869a 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -423,7 +423,11 @@ export abstract class BrowserCrawler<protected override async _cleanupContext(crawlingContext: Context): Promise<void> {const { page } = crawlingContext;- await page.close().catch((error: Error) => this.log.debug('Error while closing page', { error }));++ // Page creation may be aborted+ if (page) {+ await page.close().catch((error: Error) => this.log.debug('Error while closing page', { error }));+ }}/**diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 01155059d..d1d152534 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -201,6 +201,12 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');}+ // `browserPlugins` is working when it's not overriden by `launchContext`,+ // which for crawlers it is always overriden. Hence the error to use the other option.+ if (browserPoolOptions.browserPlugins) {+ throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');+ }+if (headless != null) {launchContext.launchOptions ??= {} as LaunchOptions;launchContext.launchOptions.headless = headless;diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex f887bdd80..1edac4a73 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -151,6 +151,12 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');}+ // `browserPlugins` is working when it's not overriden by `launchContext`,+ // which for crawlers it is always overriden. Hence the error to use the other option.+ if (browserPoolOptions.browserPlugins) {+ throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');+ }+if (headless != null) {launchContext.launchOptions ??= {} as LaunchOptions;launchContext.launchOptions.headless = headless;diff --git a/test/e2e/automatic-persist-value/actor/Dockerfile b/test/e2e/automatic-persist-value/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/automatic-persist-value/actor/Dockerfile+++ b/test/e2e/automatic-persist-value/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/automatic-persist-value/actor/main.js b/test/e2e/automatic-persist-value/actor/main.jsindex f69a17745..5ab2120b3 100644--- a/test/e2e/automatic-persist-value/actor/main.js+++ b/test/e2e/automatic-persist-value/actor/main.js@@ -8,7 +8,7 @@ const mainOptions = {};await Actor.main(async () => {- const kv = await KeyValueStore.open('test');+ const kv = await KeyValueStore.open();const crawler = new BasicCrawler({async requestHandler() {diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjsindex 405945620..b5bb21265 100644--- a/test/e2e/automatic-persist-value/test.mjs+++ b/test/e2e/automatic-persist-value/test.mjs@@ -3,15 +3,17 @@ import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);-const { keyValueStoreItems } = await runActor(testActorDirname);+const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);-await expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');+await expect(items !== undefined, 'Key value store exists');-const [{ name, raw }] = keyValueStoreItems;+await expect(items.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');-await expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');+const item = items.find((kvItem) => kvItem.name === 'crawlee');-const parsed = JSON.parse(raw.toString());+await expect(item !== undefined, 'Key-value store auto-saved value is named "crawlee"');++const parsed = JSON.parse(item.raw.toString());await expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');await expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-default-ts/actor/Dockerfile b/test/e2e/cheerio-default-ts/actor/Dockerfileindex 4ca49d27c..d7821d8de 100644--- a/test/e2e/cheerio-default-ts/actor/Dockerfile+++ b/test/e2e/cheerio-default-ts/actor/Dockerfile@@ -16,6 +16,7 @@ COPY --from=builder /usr/src/app/main.js ./# install only prod depsRUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.jsonindex 19ab0162e..aa93f767c 100644--- a/test/e2e/cheerio-default-ts/actor/package.json+++ b/test/e2e/cheerio-default-ts/actor/package.json@@ -17,6 +17,7 @@"overrides": {"apify": {"@crawlee/core": "file:./packages/core",+ "@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"}},diff --git a/test/e2e/cheerio-default/actor/Dockerfile b/test/e2e/cheerio-default/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-default/actor/Dockerfile+++ b/test/e2e/cheerio-default/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-enqueue-links/actor/Dockerfile b/test/e2e/cheerio-enqueue-links/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-enqueue-links/actor/Dockerfile+++ b/test/e2e/cheerio-enqueue-links/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-initial-cookies/actor/Dockerfile b/test/e2e/cheerio-initial-cookies/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-initial-cookies/actor/Dockerfile+++ b/test/e2e/cheerio-initial-cookies/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-max-requests/actor/Dockerfile b/test/e2e/cheerio-max-requests/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-max-requests/actor/Dockerfile+++ b/test/e2e/cheerio-max-requests/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-page-info/actor/Dockerfile b/test/e2e/cheerio-page-info/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-page-info/actor/Dockerfile+++ b/test/e2e/cheerio-page-info/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/input-json5/actor/Dockerfile+++ b/test/e2e/input-json5/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/input-json5/actor/main.js b/test/e2e/input-json5/actor/main.jsindex c991cdc85..038930b99 100644--- a/test/e2e/input-json5/actor/main.js+++ b/test/e2e/input-json5/actor/main.js@@ -1,5 +1,3 @@-import { join } from 'node:path';-import { readFileSync } from 'node:fs';import { Actor, Dataset, KeyValueStore } from 'apify';import { ApifyStorageLocal } from '@apify/storage-local';@@ -8,11 +6,10 @@ const mainOptions = {storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,};-await Actor.setValue('INPUT', readFileSync(join(process.cwd(), 'INPUT')).toString(), {- contentType: 'application/json',-});-await Actor.main(async () => {- await Dataset.pushData(await KeyValueStore.getInput());- await Dataset.pushData(await Actor.getInput());+ const a = await KeyValueStore.getInput();++ console.log(a);++ await Dataset.pushData(a);}, mainOptions);diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.jsonindex 1a91ede4b..3db16f201 100644--- a/test/e2e/input-json5/actor/package.json+++ b/test/e2e/input-json5/actor/package.json@@ -4,7 +4,9 @@"description": "JSON5 input test","dependencies": {"apify": "next",- "@apify/storage-local": "^2.1.0"+ "@apify/storage-local": "^2.1.0",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"},"overrides": {"apify": {diff --git a/test/e2e/input-json5/test.mjs b/test/e2e/input-json5/test.mjsindex 1734b68b1..715dfab4f 100644--- a/test/e2e/input-json5/test.mjs+++ b/test/e2e/input-json5/test.mjs@@ -1,16 +1,17 @@-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';+import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';++if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {+ skipTest('not supported on platform');+}const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);const { datasetItems } = await runActor(testActorDirname);-await expect(datasetItems.length === 2, 'Number of dataset items');+await expect(datasetItems.length === 1, 'Number of dataset items');await expect(JSON.stringify(datasetItems) === JSON.stringify([{hello: 'world',},- {- hello: 'world',- },]), 'Dataset items validation');diff --git a/test/e2e/jsdom-default-ts/actor/Dockerfile b/test/e2e/jsdom-default-ts/actor/Dockerfileindex 4ca49d27c..d7821d8de 100644--- a/test/e2e/jsdom-default-ts/actor/Dockerfile+++ b/test/e2e/jsdom-default-ts/actor/Dockerfile@@ -16,6 +16,7 @@ COPY --from=builder /usr/src/app/main.js ./# install only prod depsRUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.jsonindex cc1013b65..8cac752eb 100644--- a/test/e2e/jsdom-default-ts/actor/package.json+++ b/test/e2e/jsdom-default-ts/actor/package.json@@ -17,6 +17,7 @@"overrides": {"apify": {"@crawlee/core": "file:./packages/core",+ "@crawlee/types": "file:./packages/types","@crawlee/utils": "file:./packages/utils"}},diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfileindex 80883c764..de06663ad 100644--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-playwright-chrome:16-betadiff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfileindex 80883c764..de06663ad 100644--- a/test/e2e/playwright-default/actor/Dockerfile+++ b/test/e2e/playwright-default/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-playwright-chrome:16-betadiff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfileindex 80883c764..de06663ad 100644--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-playwright-chrome:16-betadiff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfileindex f9c201f3a..8a1c0fff0 100644--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-playwright-firefox:16-betadiff --git a/test/e2e/playwright-firefox-experimental-containers/actor/main.js b/test/e2e/playwright-firefox-experimental-containers/actor/main.jsindex 4b83f5295..15b81cbdc 100644--- a/test/e2e/playwright-firefox-experimental-containers/actor/main.js+++ b/test/e2e/playwright-firefox-experimental-containers/actor/main.js@@ -10,13 +10,9 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PlaywrightCrawler({- browserPoolOptions: {- browserPlugins: [- playwright.firefox,- ],- },proxyConfiguration: await Actor.createProxyConfiguration(),launchContext: {+ launcher: playwright.firefox,experimentalContainers: true,},preNavigationHooks: [(_ctx, goToOptions) => {diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfileindex 80883c764..de06663ad 100644--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-playwright-chrome:16-betadiff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/proxy-rotation/actor/Dockerfile+++ b/test/e2e/proxy-rotation/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-default/actor/Dockerfile+++ b/test/e2e/puppeteer-default/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-page-info/actor/Dockerfile+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfileindex 31f7386b7..7c7d30b82 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile@@ -3,7 +3,8 @@ FROM node:16 AS builderCOPY /packages ./packagesCOPY /package*.json ./RUN npm --quiet set progress=false \- && npm install --only=prod --no-optional+ && npm install --only=prod --no-optional \+ && npm updateFROM apify/actor-node-puppeteer-chrome:16-betadiff --git a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-zero-concurrency/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile+++ b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/request-skip-navigation/actor/Dockerfile b/test/e2e/request-skip-navigation/actor/Dockerfileindex 26898a6e1..b5c27460b 100644--- a/test/e2e/request-skip-navigation/actor/Dockerfile+++ b/test/e2e/request-skip-navigation/actor/Dockerfile@@ -5,6 +5,7 @@ COPY package*.json ./RUN npm --quiet set progress=false \&& npm install --only=prod --no-optional \+ && npm update \&& echo "Installed NPM packages:" \&& (npm list --only=prod --no-optional --all || true) \&& echo "Node.js version:" \diff --git a/test/e2e/run.mjs b/test/e2e/run.mjsindex 70f3fc66c..622416aa3 100644--- a/test/e2e/run.mjs+++ b/test/e2e/run.mjs@@ -1,4 +1,4 @@-import { dirname, join } from 'node:path';+import { dirname } from 'node:path';import { fileURLToPath } from 'node:url';import { once } from 'node:events';import { readdir } from 'node:fs/promises';@@ -81,8 +81,8 @@ async function run() {return;}- if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM' && (str.startsWith('[build]') || str.startsWith('[run]'))) {- const platformStatsMessage = str.match(/\[(?:run|build)] (.*)/);+ if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM' && (str.startsWith('[build]') || str.startsWith('[run]') || str.startsWith('[kv]'))) {+ const platformStatsMessage = str.match(/\[(?:run|build|kv)] (.*)/);if (platformStatsMessage) {console.log(`${colors.yellow(`[${dir.name}] `)}${colors.grey(platformStatsMessage[1])}`);}@@ -131,9 +131,5 @@ if (isMainThread) {// We want to exit with non-zero code if any of the tests failedif (failure) process.exit(1);} else {- // Mimic CWD for proper INPUT loading- // See packages/core/src/storages/key_value_store.ts- process.cwd = () => join(basePath, workerData);-await import(`${basePath}/${workerData}/test.mjs`);}diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex 96427f0d1..1603168b9 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -1,15 +1,21 @@-import { dirname, join, resolve } from 'node:path';+import { dirname, join } from 'node:path';import { fileURLToPath } from 'node:url';import { existsSync } from 'node:fs';import { readdir, readFile } from 'node:fs/promises';import { homedir } from 'node:os';import { setTimeout } from 'node:timers/promises';import { execSync } from 'node:child_process';+import { got } from 'got';import fs from 'fs-extra';import { Actor } from 'apify';// eslint-disable-next-line import/no-relative-packagesimport { URL_NO_COMMAS_REGEX } from '../../packages/utils/dist/index.mjs';+/**+ * @param {string} name+ */+const isPrivateEntry = (name) => name === 'SDK_CRAWLER_STATISTICS_0' || name === 'SDK_SESSION_POOL_STATE';+export const SKIPPED_TEST_CLOSE_CODE = 404;/** @type {Record<string, (text: string) => string>} */@@ -60,53 +66,105 @@ export function getActorTestDir(url) {export async function runActor(dirName, memory = 4096) {let stats;let datasetItems;- let keyValueStoreItems;+ let getKeyValueStoreItems;+ let defaultKeyValueStoreItems;++ const inputPath = join(dirName, '..', 'INPUT');+ const input = fs.existsSync(inputPath) ? fs.readFileSync(inputPath) : undefined;+ const contentType = input ? 'application/json' : undefined;if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {+ const client = Actor.newClient();+await copyPackages(dirName);execSync('npx -y apify-cli push', { cwd: dirName });const actorName = await getActorName(dirName);- const client = Actor.newClient();const { items: actors } = await client.actors().list();const { id } = actors.find((actor) => actor.name === actorName);+ const gotClient = got.extend({+ retry: {+ limit: 0,+ },+ headers: {+ 'user-agent': 'crawlee e2e tests (got)',+ },+ timeout: {+ request: 10000,+ },+ });++ // Do NOT use Apify Client yet!+ // See https://github.com/apify/apify-client-js/issues/277+ const { data: { id: runId } } = await gotClient(`https://api.apify.com/v2/acts/${id}/runs`, {+ method: 'POST',+ searchParams: {+ token: client.token,+ memory,+ },+ headers: {+ 'content-type': contentType,+ },+ body: input,+ retry: {+ limit: 0,+ },+ }).json();+const {defaultKeyValueStoreId,defaultDatasetId,startedAt: runStartedAt,finishedAt: runFinishedAt,- id: runId,+ // id: runId,buildId,- } = await client.actor(id).call(null, { memory });+ userId,+ } = await client.run(runId).waitForFinish();++ getKeyValueStoreItems = async (name) => {+ const kvResult = await client.keyValueStore(name ? `${userId}/${name}` : defaultKeyValueStoreId).get();++ if (kvResult) {+ const { items: keyValueItems } = await client.keyValueStore(kvResult.id).listKeys();++ if (keyValueItems.length) {+ console.log(`[kv] View storage: https://console.apify.com/storage/key-value/${kvResult.id}`);+ }++ const entries = await Promise.all(keyValueItems.map(async ({ key }) => {+ const record = await client.keyValueStore(kvResult.id).getRecord(key, { buffer: true });++ return {+ name: record.key,+ raw: record.value,+ };+ }));++ return entries.filter(({ name }) => !isPrivateEntry(name));+ }++ return undefined;+ };+const {startedAt: buildStartedAt,finishedAt: buildFinishedAt,} = await client.build(buildId).get();+const buildTook = (buildFinishedAt.getTime() - buildStartedAt.getTime()) / 1000;console.log(`[build] View build log: https://api.apify.com/v2/logs/${buildId} [build took ${buildTook}s]`);+const runTook = (runFinishedAt.getTime() - runStartedAt.getTime()) / 1000;console.log(`[run] View run: https://console.apify.com/view/runs/${runId} [run took ${runTook}s]`);- const { value } = await client.keyValueStore(defaultKeyValueStoreId).getRecord('SDK_CRAWLER_STATISTICS_0');- stats = value;+ const statsRecord = await client.keyValueStore(defaultKeyValueStoreId).getRecord('SDK_CRAWLER_STATISTICS_0');+ stats = statsRecord?.value;+const { items } = await client.dataset(defaultDatasetId).listItems();datasetItems = items;- try {- const { items: keyValueItems } = await client.keyValueStore('test').listKeys();-- keyValueStoreItems = await Promise.all(keyValueItems.map(async ({ key }) => {- const record = await client.keyValueStore('test').getRecord(key, { buffer: true });-- return {- name: record.key,- raw: record.value,- };- }));- } catch {- keyValueStoreItems = [];- }+ defaultKeyValueStoreItems = await getKeyValueStoreItems();} else {if (dirName.split('/').at(-2).endsWith('-ts')) {try {@@ -122,15 +180,26 @@ export async function runActor(dirName, memory = 4096) {}}+ if (input) {+ await Actor.setValue('INPUT', input, { contentType });+ }+await import(join(dirName, 'main.js'));await setTimeout(50);stats = await getStats(dirName);datasetItems = await getDatasetItems(dirName);- keyValueStoreItems = await getKeyValueStoreItems(dirName);++ getKeyValueStoreItems = (name = 'default') => getLocalKeyValueStoreItems(dirName, name);+ defaultKeyValueStoreItems = await getKeyValueStoreItems();}- return { stats, datasetItems, keyValueStoreItems };+ return {+ stats,+ datasetItems,+ defaultKeyValueStoreItems,+ getKeyValueStoreItems,+ };}/**@@ -150,11 +219,15 @@ async function getActorName(dirName) {* @internal*/async function copyPackages(dirName) {- const srcPackagesDir = resolve('./', 'packages');+ const srcPackagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'packages');const destPackagesDir = join(dirName, 'packages');await fs.remove(destPackagesDir);- const { dependencies } = await fs.readJSON(join(dirName, 'package.json'));+ const { dependencies, overrides } = await fs.readJSON(join(dirName, 'package.json'));++ if (overrides?.apify) {+ Object.assign(dependencies, overrides.apify);+ }// We don't need to copy the following packagesdelete dependencies['@apify/storage-local'];@@ -236,15 +309,16 @@ export async function getDatasetItems(dirName) {}/**- * Gets all items in the key-value store, as a Buffer+ * Gets all items in the local key-value store* @param {string} dirName+ * @param {string} kvName*/-export async function getKeyValueStoreItems(dirName) {+export async function getLocalKeyValueStoreItems(dirName, kvName) {const dir = getStorage(dirName);- const storePath = join(dir, `key_value_stores/test/`);+ const storePath = join(dir, 'key_value_stores', kvName);if (!existsSync(storePath)) {- return [];+ return undefined;}const dirents = await readdir(storePath, { withFileTypes: true });@@ -257,7 +331,13 @@ export async function getKeyValueStoreItems(dirName) {const filePath = join(storePath, fileName.name);const buffer = await readFile(filePath);- keyValueStoreRecords.push({ name: fileName.name.split('.').slice(0, -1).join('.'), raw: buffer });+ const name = fileName.name.split('.').slice(0, -1).join('.');++ if (isPrivateEntry(name)) {+ continue;+ }++ keyValueStoreRecords.push({ name, raw: buffer });}return keyValueStoreRecords;