Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: fix PLATFORM e2e tests (#1558)

Szymon Marczak73e105a2654a
diff --git a/package-lock.json b/package-lock.json
index 448e81d38..82bb1c420 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -46,6 +46,7 @@
"fs-extra": "^10.1.0",
"gen-esm-wrapper": "^1.1.3",
"globby": "^13.1.2",
+ "got": "^12.5.0",
"husky": "^8.0.1",
"jest": "^29.1.2",
"lerna": "^5.5.4",
@@ -8652,6 +8653,31 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
+ "node_modules/got": {
+ "version": "12.5.0",
+ "resolved": "https://registry.npmjs.org/got/-/got-12.5.0.tgz",
+ "integrity": "sha512-/Bneo/L6bLN1wDyJCeRZ3CLoixvwb9v3rE3IHulFSfTHwP85xSr4QatA8K0c6GlL5+mc4IZ57BzluNZJiXvHIg==",
+ "dev": true,
+ "dependencies": {
+ "@sindresorhus/is": "^5.2.0",
+ "@szmarczak/http-timer": "^5.0.1",
+ "cacheable-lookup": "^6.0.4",
+ "cacheable-request": "^10.1.2",
+ "decompress-response": "^6.0.0",
+ "form-data-encoder": "^2.1.2",
+ "get-stream": "^6.0.1",
+ "http2-wrapper": "^2.1.10",
+ "lowercase-keys": "^3.0.0",
+ "p-cancelable": "^3.0.0",
+ "responselike": "^3.0.0"
+ },
+ "engines": {
+ "node": ">=14.16"
+ },
+ "funding": {
+ "url": "https://github.com/sindresorhus/got?sponsor=1"
+ }
+ },
"node_modules/got-cjs": {
"version": "12.3.1",
"resolved": "https://registry.npmjs.org/got-cjs/-/got-cjs-12.3.1.tgz",
@@ -8695,6 +8721,116 @@
"node": ">=15.10.0"
}
},
+ "node_modules/got/node_modules/@sindresorhus/is": {
+ "version": "5.3.0",
+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz",
+ "integrity": "sha512-CX6t4SYQ37lzxicAqsBtxA3OseeoVrh9cSJ5PFYam0GksYlupRfy1A+Q4aYD3zvcfECLc0zO2u+ZnR2UYKvCrw==",
+ "dev": true,
+ "engines": {
+ "node": ">=14.16"
+ },
+ "funding": {
+ "url": "https://github.com/sindresorhus/is?sponsor=1"
+ }
+ },
+ "node_modules/got/node_modules/@szmarczak/http-timer": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
+ "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
+ "dev": true,
+ "dependencies": {
+ "defer-to-connect": "^2.0.1"
+ },
+ "engines": {
+ "node": ">=14.16"
+ }
+ },
+ "node_modules/got/node_modules/cacheable-request": {
+ "version": "10.2.0",
+ "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.0.tgz",
+ "integrity": "sha512-PLOMmzyOU3L5HECdZC0PTNawPdP/taOhCNrYUmZchrw7gwERMctfx2wZcndb63w5W06PTHC1Poq89J9chMZPLw==",
+ "dev": true,
+ "dependencies": {
+ "get-stream": "^6.0.1",
+ "http-cache-semantics": "^4.1.0",
+ "keyv": "^4.5.0",
+ "mimic-response": "^4.0.0",
+ "normalize-url": "^7.1.0",
+ "responselike": "^3.0.0"
+ },
+ "engines": {
+ "node": ">=14.16"
+ }
+ },
+ "node_modules/got/node_modules/form-data-encoder": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.2.tgz",
+ "integrity": "sha512-FCaIOVTRA9E0siY6FeXid7D5yrCqpsErplUkE2a1BEiKj1BE9z6FbKB4ntDTwC4NVLie9p+4E9nX4mWwEOT05A==",
+ "dev": true,
+ "engines": {
+ "node": ">= 14.17"
+ }
+ },
+ "node_modules/got/node_modules/lowercase-keys": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
+ "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
+ "dev": true,
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/got/node_modules/mimic-response": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
+ "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
+ "dev": true,
+ "engines": {
+ "node": "^12.20.0 || ^14.13.1 || >=16.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/got/node_modules/normalize-url": {
+ "version": "7.1.0",
+ "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.1.0.tgz",
+ "integrity": "sha512-JgkdydFdLe1E5Q7DpLvKVyBZOOwXYGhIbMbOMm3lJ06XKzaiit+qo1HciO3z3IFklStfarzJHVQf9ZcNPTvZlw==",
+ "dev": true,
+ "engines": {
+ "node": ">=12.20"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
+ "node_modules/got/node_modules/p-cancelable": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
+ "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==",
+ "dev": true,
+ "engines": {
+ "node": ">=12.20"
+ }
+ },
+ "node_modules/got/node_modules/responselike": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
+ "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
+ "dev": true,
+ "dependencies": {
+ "lowercase-keys": "^3.0.0"
+ },
+ "engines": {
+ "node": ">=14.16"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/sindresorhus"
+ }
+ },
"node_modules/graceful-fs": {
"version": "4.2.10",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz",
@@ -24100,6 +24236,95 @@
"slash": "^4.0.0"
}
},
+ "got": {
+ "version": "12.5.0",
+ "resolved": "https://registry.npmjs.org/got/-/got-12.5.0.tgz",
+ "integrity": "sha512-/Bneo/L6bLN1wDyJCeRZ3CLoixvwb9v3rE3IHulFSfTHwP85xSr4QatA8K0c6GlL5+mc4IZ57BzluNZJiXvHIg==",
+ "dev": true,
+ "requires": {
+ "@sindresorhus/is": "^5.2.0",
+ "@szmarczak/http-timer": "^5.0.1",
+ "cacheable-lookup": "^6.0.4",
+ "cacheable-request": "^10.1.2",
+ "decompress-response": "^6.0.0",
+ "form-data-encoder": "^2.1.2",
+ "get-stream": "^6.0.1",
+ "http2-wrapper": "^2.1.10",
+ "lowercase-keys": "^3.0.0",
+ "p-cancelable": "^3.0.0",
+ "responselike": "^3.0.0"
+ },
+ "dependencies": {
+ "@sindresorhus/is": {
+ "version": "5.3.0",
+ "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-5.3.0.tgz",
+ "integrity": "sha512-CX6t4SYQ37lzxicAqsBtxA3OseeoVrh9cSJ5PFYam0GksYlupRfy1A+Q4aYD3zvcfECLc0zO2u+ZnR2UYKvCrw==",
+ "dev": true
+ },
+ "@szmarczak/http-timer": {
+ "version": "5.0.1",
+ "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-5.0.1.tgz",
+ "integrity": "sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==",
+ "dev": true,
+ "requires": {
+ "defer-to-connect": "^2.0.1"
+ }
+ },
+ "cacheable-request": {
+ "version": "10.2.0",
+ "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-10.2.0.tgz",
+ "integrity": "sha512-PLOMmzyOU3L5HECdZC0PTNawPdP/taOhCNrYUmZchrw7gwERMctfx2wZcndb63w5W06PTHC1Poq89J9chMZPLw==",
+ "dev": true,
+ "requires": {
+ "get-stream": "^6.0.1",
+ "http-cache-semantics": "^4.1.0",
+ "keyv": "^4.5.0",
+ "mimic-response": "^4.0.0",
+ "normalize-url": "^7.1.0",
+ "responselike": "^3.0.0"
+ }
+ },
+ "form-data-encoder": {
+ "version": "2.1.2",
+ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.2.tgz",
+ "integrity": "sha512-FCaIOVTRA9E0siY6FeXid7D5yrCqpsErplUkE2a1BEiKj1BE9z6FbKB4ntDTwC4NVLie9p+4E9nX4mWwEOT05A==",
+ "dev": true
+ },
+ "lowercase-keys": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-3.0.0.tgz",
+ "integrity": "sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==",
+ "dev": true
+ },
+ "mimic-response": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-4.0.0.tgz",
+ "integrity": "sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==",
+ "dev": true
+ },
+ "normalize-url": {
+ "version": "7.1.0",
+ "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-7.1.0.tgz",
+ "integrity": "sha512-JgkdydFdLe1E5Q7DpLvKVyBZOOwXYGhIbMbOMm3lJ06XKzaiit+qo1HciO3z3IFklStfarzJHVQf9ZcNPTvZlw==",
+ "dev": true
+ },
+ "p-cancelable": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz",
+ "integrity": "sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==",
+ "dev": true
+ },
+ "responselike": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/responselike/-/responselike-3.0.0.tgz",
+ "integrity": "sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==",
+ "dev": true,
+ "requires": {
+ "lowercase-keys": "^3.0.0"
+ }
+ }
+ }
+ },
"got-cjs": {
"version": "12.3.1",
"resolved": "https://registry.npmjs.org/got-cjs/-/got-cjs-12.3.1.tgz",
diff --git a/package.json b/package.json
index 19a9f1091..ea9566231 100644
--- a/package.json
+++ b/package.json
@@ -84,6 +84,7 @@
"fs-extra": "^10.1.0",
"gen-esm-wrapper": "^1.1.3",
"globby": "^13.1.2",
+ "got": "^12.5.0",
"husky": "^8.0.1",
"jest": "^29.1.2",
"lerna": "^5.5.4",
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index af891e760..539a4869a 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -423,7 +423,11 @@ export abstract class BrowserCrawler<
protected override async _cleanupContext(crawlingContext: Context): Promise<void> {
const { page } = crawlingContext;
- await page.close().catch((error: Error) => this.log.debug('Error while closing page', { error }));
+
+ // Page creation may be aborted
+ if (page) {
+ await page.close().catch((error: Error) => this.log.debug('Error while closing page', { error }));
+ }
}
/**
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 01155059d..d1d152534 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -201,6 +201,12 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri
+ 'Use PlaywrightCrawlerOptions.proxyConfiguration');
}
+ // `browserPlugins` is working when it's not overriden by `launchContext`,
+ // which for crawlers it is always overriden. Hence the error to use the other option.
+ if (browserPoolOptions.browserPlugins) {
+ throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
+ }
+
if (headless != null) {
launchContext.launchOptions ??= {} as LaunchOptions;
launchContext.launchOptions.headless = headless;
diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
index f887bdd80..1edac4a73 100644
--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts
@@ -151,6 +151,12 @@ export class PuppeteerCrawler extends BrowserCrawler<{ browserPlugins: [Puppetee
+ 'Use PuppeteerCrawlerOptions.proxyConfiguration');
}
+ // `browserPlugins` is working when it's not overriden by `launchContext`,
+ // which for crawlers it is always overriden. Hence the error to use the other option.
+ if (browserPoolOptions.browserPlugins) {
+ throw new Error('browserPoolOptions.browserPlugins is disallowed. Use launchContext.launcher instead.');
+ }
+
if (headless != null) {
launchContext.launchOptions ??= {} as LaunchOptions;
launchContext.launchOptions.headless = headless;
diff --git a/test/e2e/automatic-persist-value/actor/Dockerfile b/test/e2e/automatic-persist-value/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/automatic-persist-value/actor/Dockerfile
+++ b/test/e2e/automatic-persist-value/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/automatic-persist-value/actor/main.js b/test/e2e/automatic-persist-value/actor/main.js
index f69a17745..5ab2120b3 100644
--- a/test/e2e/automatic-persist-value/actor/main.js
+++ b/test/e2e/automatic-persist-value/actor/main.js
@@ -8,7 +8,7 @@ const mainOptions = {
};
await Actor.main(async () => {
- const kv = await KeyValueStore.open('test');
+ const kv = await KeyValueStore.open();
const crawler = new BasicCrawler({
async requestHandler() {
diff --git a/test/e2e/automatic-persist-value/test.mjs b/test/e2e/automatic-persist-value/test.mjs
index 405945620..b5bb21265 100644
--- a/test/e2e/automatic-persist-value/test.mjs
+++ b/test/e2e/automatic-persist-value/test.mjs
@@ -3,15 +3,17 @@ import { initialize, expect, getActorTestDir, runActor } from '../tools.mjs';
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
-const { keyValueStoreItems } = await runActor(testActorDirname);
+const { defaultKeyValueStoreItems: items } = await runActor(testActorDirname);
-await expect(keyValueStoreItems.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');
+await expect(items !== undefined, 'Key value store exists');
-const [{ name, raw }] = keyValueStoreItems;
+await expect(items.length === 1, 'Key-value store automatically saved the value expected to be auto-saved');
-await expect(name === 'crawlee', 'Key-value store auto-saved value is named "crawlee"');
+const item = items.find((kvItem) => kvItem.name === 'crawlee');
-const parsed = JSON.parse(raw.toString());
+await expect(item !== undefined, 'Key-value store auto-saved value is named "crawlee"');
+
+const parsed = JSON.parse(item.raw.toString());
await expect(typeof parsed === 'object' && parsed !== null, 'Key-value store auto-save value is a non-nullable object');
await expect(parsed.crawlee === 'awesome!', 'Key-value store auto-save value has a property "crawlee" that is set to "awesome!"');
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile
+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-default-ts/actor/Dockerfile b/test/e2e/cheerio-default-ts/actor/Dockerfile
index 4ca49d27c..d7821d8de 100644
--- a/test/e2e/cheerio-default-ts/actor/Dockerfile
+++ b/test/e2e/cheerio-default-ts/actor/Dockerfile
@@ -16,6 +16,7 @@ COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-default-ts/actor/package.json b/test/e2e/cheerio-default-ts/actor/package.json
index 19ab0162e..aa93f767c 100644
--- a/test/e2e/cheerio-default-ts/actor/package.json
+++ b/test/e2e/cheerio-default-ts/actor/package.json
@@ -17,6 +17,7 @@
"overrides": {
"apify": {
"@crawlee/core": "file:./packages/core",
+ "@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
}
},
diff --git a/test/e2e/cheerio-default/actor/Dockerfile b/test/e2e/cheerio-default/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-default/actor/Dockerfile
+++ b/test/e2e/cheerio-default/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-enqueue-links/actor/Dockerfile b/test/e2e/cheerio-enqueue-links/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-enqueue-links/actor/Dockerfile
+++ b/test/e2e/cheerio-enqueue-links/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-initial-cookies/actor/Dockerfile b/test/e2e/cheerio-initial-cookies/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-initial-cookies/actor/Dockerfile
+++ b/test/e2e/cheerio-initial-cookies/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-max-requests/actor/Dockerfile b/test/e2e/cheerio-max-requests/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-max-requests/actor/Dockerfile
+++ b/test/e2e/cheerio-max-requests/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-page-info/actor/Dockerfile b/test/e2e/cheerio-page-info/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-page-info/actor/Dockerfile
+++ b/test/e2e/cheerio-page-info/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/input-json5/actor/Dockerfile b/test/e2e/input-json5/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/input-json5/actor/Dockerfile
+++ b/test/e2e/input-json5/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/input-json5/actor/main.js b/test/e2e/input-json5/actor/main.js
index c991cdc85..038930b99 100644
--- a/test/e2e/input-json5/actor/main.js
+++ b/test/e2e/input-json5/actor/main.js
@@ -1,5 +1,3 @@
-import { join } from 'node:path';
-import { readFileSync } from 'node:fs';
import { Actor, Dataset, KeyValueStore } from 'apify';
import { ApifyStorageLocal } from '@apify/storage-local';
@@ -8,11 +6,10 @@ const mainOptions = {
storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
};
-await Actor.setValue('INPUT', readFileSync(join(process.cwd(), 'INPUT')).toString(), {
- contentType: 'application/json',
-});
-
await Actor.main(async () => {
- await Dataset.pushData(await KeyValueStore.getInput());
- await Dataset.pushData(await Actor.getInput());
+ const a = await KeyValueStore.getInput();
+
+ console.log(a);
+
+ await Dataset.pushData(a);
}, mainOptions);
diff --git a/test/e2e/input-json5/actor/package.json b/test/e2e/input-json5/actor/package.json
index 1a91ede4b..3db16f201 100644
--- a/test/e2e/input-json5/actor/package.json
+++ b/test/e2e/input-json5/actor/package.json
@@ -4,7 +4,9 @@
"description": "JSON5 input test",
"dependencies": {
"apify": "next",
- "@apify/storage-local": "^2.1.0"
+ "@apify/storage-local": "^2.1.0",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
},
"overrides": {
"apify": {
diff --git a/test/e2e/input-json5/test.mjs b/test/e2e/input-json5/test.mjs
index 1734b68b1..715dfab4f 100644
--- a/test/e2e/input-json5/test.mjs
+++ b/test/e2e/input-json5/test.mjs
@@ -1,16 +1,17 @@
-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';
+
+if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {
+ skipTest('not supported on platform');
+}
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
const { datasetItems } = await runActor(testActorDirname);
-await expect(datasetItems.length === 2, 'Number of dataset items');
+await expect(datasetItems.length === 1, 'Number of dataset items');
await expect(JSON.stringify(datasetItems) === JSON.stringify([
{
hello: 'world',
},
- {
- hello: 'world',
- },
]), 'Dataset items validation');
diff --git a/test/e2e/jsdom-default-ts/actor/Dockerfile b/test/e2e/jsdom-default-ts/actor/Dockerfile
index 4ca49d27c..d7821d8de 100644
--- a/test/e2e/jsdom-default-ts/actor/Dockerfile
+++ b/test/e2e/jsdom-default-ts/actor/Dockerfile
@@ -16,6 +16,7 @@ COPY --from=builder /usr/src/app/main.js ./
# install only prod deps
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/jsdom-default-ts/actor/package.json b/test/e2e/jsdom-default-ts/actor/package.json
index cc1013b65..8cac752eb 100644
--- a/test/e2e/jsdom-default-ts/actor/package.json
+++ b/test/e2e/jsdom-default-ts/actor/package.json
@@ -17,6 +17,7 @@
"overrides": {
"apify": {
"@crawlee/core": "file:./packages/core",
+ "@crawlee/types": "file:./packages/types",
"@crawlee/utils": "file:./packages/utils"
}
},
diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
index 80883c764..de06663ad 100644
--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-playwright-chrome:16-beta
diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfile
index 80883c764..de06663ad 100644
--- a/test/e2e/playwright-default/actor/Dockerfile
+++ b/test/e2e/playwright-default/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-playwright-chrome:16-beta
diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfile
index 80883c764..de06663ad 100644
--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile
+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-playwright-chrome:16-beta
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
index f9c201f3a..8a1c0fff0 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-playwright-firefox:16-beta
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/main.js b/test/e2e/playwright-firefox-experimental-containers/actor/main.js
index 4b83f5295..15b81cbdc 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/main.js
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/main.js
@@ -10,13 +10,9 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PlaywrightCrawler({
- browserPoolOptions: {
- browserPlugins: [
- playwright.firefox,
- ],
- },
proxyConfiguration: await Actor.createProxyConfiguration(),
launchContext: {
+ launcher: playwright.firefox,
experimentalContainers: true,
},
preNavigationHooks: [(_ctx, goToOptions) => {
diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile
index 80883c764..de06663ad 100644
--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile
+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-playwright-chrome:16-beta
diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/proxy-rotation/actor/Dockerfile
+++ b/test/e2e/proxy-rotation/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-default/actor/Dockerfile
+++ b/test/e2e/puppeteer-default/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-page-info/actor/Dockerfile
+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
index 31f7386b7..7c7d30b82 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
@@ -3,7 +3,8 @@ FROM node:16 AS builder
COPY /packages ./packages
COPY /package*.json ./
RUN npm --quiet set progress=false \
- && npm install --only=prod --no-optional
+ && npm install --only=prod --no-optional \
+ && npm update
FROM apify/actor-node-puppeteer-chrome:16-beta
diff --git a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/request-queue-zero-concurrency/actor/Dockerfile
+++ b/test/e2e/request-queue-zero-concurrency/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/request-skip-navigation/actor/Dockerfile b/test/e2e/request-skip-navigation/actor/Dockerfile
index 26898a6e1..b5c27460b 100644
--- a/test/e2e/request-skip-navigation/actor/Dockerfile
+++ b/test/e2e/request-skip-navigation/actor/Dockerfile
@@ -5,6 +5,7 @@ COPY package*.json ./
RUN npm --quiet set progress=false \
&& npm install --only=prod --no-optional \
+ && npm update \
&& echo "Installed NPM packages:" \
&& (npm list --only=prod --no-optional --all || true) \
&& echo "Node.js version:" \
diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs
index 70f3fc66c..622416aa3 100644
--- a/test/e2e/run.mjs
+++ b/test/e2e/run.mjs
@@ -1,4 +1,4 @@
-import { dirname, join } from 'node:path';
+import { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { once } from 'node:events';
import { readdir } from 'node:fs/promises';
@@ -81,8 +81,8 @@ async function run() {
return;
}
- if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM' && (str.startsWith('[build]') || str.startsWith('[run]'))) {
- const platformStatsMessage = str.match(/\[(?:run|build)] (.*)/);
+ if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM' && (str.startsWith('[build]') || str.startsWith('[run]') || str.startsWith('[kv]'))) {
+ const platformStatsMessage = str.match(/\[(?:run|build|kv)] (.*)/);
if (platformStatsMessage) {
console.log(`${colors.yellow(`[${dir.name}] `)}${colors.grey(platformStatsMessage[1])}`);
}
@@ -131,9 +131,5 @@ if (isMainThread) {
// We want to exit with non-zero code if any of the tests failed
if (failure) process.exit(1);
} else {
- // Mimic CWD for proper INPUT loading
- // See packages/core/src/storages/key_value_store.ts
- process.cwd = () => join(basePath, workerData);
-
await import(`${basePath}/${workerData}/test.mjs`);
}
diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs
index 96427f0d1..1603168b9 100644
--- a/test/e2e/tools.mjs
+++ b/test/e2e/tools.mjs
@@ -1,15 +1,21 @@
-import { dirname, join, resolve } from 'node:path';
+import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { existsSync } from 'node:fs';
import { readdir, readFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { setTimeout } from 'node:timers/promises';
import { execSync } from 'node:child_process';
+import { got } from 'got';
import fs from 'fs-extra';
import { Actor } from 'apify';
// eslint-disable-next-line import/no-relative-packages
import { URL_NO_COMMAS_REGEX } from '../../packages/utils/dist/index.mjs';
+/**
+ * @param {string} name
+ */
+const isPrivateEntry = (name) => name === 'SDK_CRAWLER_STATISTICS_0' || name === 'SDK_SESSION_POOL_STATE';
+
export const SKIPPED_TEST_CLOSE_CODE = 404;
/** @type {Record<string, (text: string) => string>} */
@@ -60,53 +66,105 @@ export function getActorTestDir(url) {
export async function runActor(dirName, memory = 4096) {
let stats;
let datasetItems;
- let keyValueStoreItems;
+ let getKeyValueStoreItems;
+ let defaultKeyValueStoreItems;
+
+ const inputPath = join(dirName, '..', 'INPUT');
+ const input = fs.existsSync(inputPath) ? fs.readFileSync(inputPath) : undefined;
+ const contentType = input ? 'application/json' : undefined;
if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {
+ const client = Actor.newClient();
+
await copyPackages(dirName);
execSync('npx -y apify-cli push', { cwd: dirName });
const actorName = await getActorName(dirName);
- const client = Actor.newClient();
const { items: actors } = await client.actors().list();
const { id } = actors.find((actor) => actor.name === actorName);
+ const gotClient = got.extend({
+ retry: {
+ limit: 0,
+ },
+ headers: {
+ 'user-agent': 'crawlee e2e tests (got)',
+ },
+ timeout: {
+ request: 10000,
+ },
+ });
+
+ // Do NOT use Apify Client yet!
+ // See https://github.com/apify/apify-client-js/issues/277
+ const { data: { id: runId } } = await gotClient(`https://api.apify.com/v2/acts/${id}/runs`, {
+ method: 'POST',
+ searchParams: {
+ token: client.token,
+ memory,
+ },
+ headers: {
+ 'content-type': contentType,
+ },
+ body: input,
+ retry: {
+ limit: 0,
+ },
+ }).json();
+
const {
defaultKeyValueStoreId,
defaultDatasetId,
startedAt: runStartedAt,
finishedAt: runFinishedAt,
- id: runId,
+ // id: runId,
buildId,
- } = await client.actor(id).call(null, { memory });
+ userId,
+ } = await client.run(runId).waitForFinish();
+
+ getKeyValueStoreItems = async (name) => {
+ const kvResult = await client.keyValueStore(name ? `${userId}/${name}` : defaultKeyValueStoreId).get();
+
+ if (kvResult) {
+ const { items: keyValueItems } = await client.keyValueStore(kvResult.id).listKeys();
+
+ if (keyValueItems.length) {
+ console.log(`[kv] View storage: https://console.apify.com/storage/key-value/${kvResult.id}`);
+ }
+
+ const entries = await Promise.all(keyValueItems.map(async ({ key }) => {
+ const record = await client.keyValueStore(kvResult.id).getRecord(key, { buffer: true });
+
+ return {
+ name: record.key,
+ raw: record.value,
+ };
+ }));
+
+ return entries.filter(({ name }) => !isPrivateEntry(name));
+ }
+
+ return undefined;
+ };
+
const {
startedAt: buildStartedAt,
finishedAt: buildFinishedAt,
} = await client.build(buildId).get();
+
const buildTook = (buildFinishedAt.getTime() - buildStartedAt.getTime()) / 1000;
console.log(`[build] View build log: https://api.apify.com/v2/logs/${buildId} [build took ${buildTook}s]`);
+
const runTook = (runFinishedAt.getTime() - runStartedAt.getTime()) / 1000;
console.log(`[run] View run: https://console.apify.com/view/runs/${runId} [run took ${runTook}s]`);
- const { value } = await client.keyValueStore(defaultKeyValueStoreId).getRecord('SDK_CRAWLER_STATISTICS_0');
- stats = value;
+ const statsRecord = await client.keyValueStore(defaultKeyValueStoreId).getRecord('SDK_CRAWLER_STATISTICS_0');
+ stats = statsRecord?.value;
+
const { items } = await client.dataset(defaultDatasetId).listItems();
datasetItems = items;
- try {
- const { items: keyValueItems } = await client.keyValueStore('test').listKeys();
-
- keyValueStoreItems = await Promise.all(keyValueItems.map(async ({ key }) => {
- const record = await client.keyValueStore('test').getRecord(key, { buffer: true });
-
- return {
- name: record.key,
- raw: record.value,
- };
- }));
- } catch {
- keyValueStoreItems = [];
- }
+ defaultKeyValueStoreItems = await getKeyValueStoreItems();
} else {
if (dirName.split('/').at(-2).endsWith('-ts')) {
try {
@@ -122,15 +180,26 @@ export async function runActor(dirName, memory = 4096) {
}
}
+ if (input) {
+ await Actor.setValue('INPUT', input, { contentType });
+ }
+
await import(join(dirName, 'main.js'));
await setTimeout(50);
stats = await getStats(dirName);
datasetItems = await getDatasetItems(dirName);
- keyValueStoreItems = await getKeyValueStoreItems(dirName);
+
+ getKeyValueStoreItems = (name = 'default') => getLocalKeyValueStoreItems(dirName, name);
+ defaultKeyValueStoreItems = await getKeyValueStoreItems();
}
- return { stats, datasetItems, keyValueStoreItems };
+ return {
+ stats,
+ datasetItems,
+ defaultKeyValueStoreItems,
+ getKeyValueStoreItems,
+ };
}
/**
@@ -150,11 +219,15 @@ async function getActorName(dirName) {
* @internal
*/
async function copyPackages(dirName) {
- const srcPackagesDir = resolve('./', 'packages');
+ const srcPackagesDir = join(dirname(fileURLToPath(import.meta.url)), '..', '..', 'packages');
const destPackagesDir = join(dirName, 'packages');
await fs.remove(destPackagesDir);
- const { dependencies } = await fs.readJSON(join(dirName, 'package.json'));
+ const { dependencies, overrides } = await fs.readJSON(join(dirName, 'package.json'));
+
+ if (overrides?.apify) {
+ Object.assign(dependencies, overrides.apify);
+ }
// We don't need to copy the following packages
delete dependencies['@apify/storage-local'];
@@ -236,15 +309,16 @@ export async function getDatasetItems(dirName) {
}
/**
- * Gets all items in the key-value store, as a Buffer
+ * Gets all items in the local key-value store
* @param {string} dirName
+ * @param {string} kvName
*/
-export async function getKeyValueStoreItems(dirName) {
+export async function getLocalKeyValueStoreItems(dirName, kvName) {
const dir = getStorage(dirName);
- const storePath = join(dir, `key_value_stores/test/`);
+ const storePath = join(dir, 'key_value_stores', kvName);
if (!existsSync(storePath)) {
- return [];
+ return undefined;
}
const dirents = await readdir(storePath, { withFileTypes: true });
@@ -257,7 +331,13 @@ export async function getKeyValueStoreItems(dirName) {
const filePath = join(storePath, fileName.name);
const buffer = await readFile(filePath);
- keyValueStoreRecords.push({ name: fileName.name.split('.').slice(0, -1).join('.'), raw: buffer });
+ const name = fileName.name.split('.').slice(0, -1).join('.');
+
+ if (isPrivateEntry(name)) {
+ continue;
+ }
+
+ keyValueStoreRecords.push({ name, raw: buffer });
}
return keyValueStoreRecords;