Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: e2e tests fix, better logging (#1713)

Jindřich Bärfd74447438e2
diff --git a/test/e2e/automatic-persist-value/actor/apify.json b/test/e2e/automatic-persist-value/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/automatic-persist-value/actor/apify.json
rename to test/e2e/automatic-persist-value/actor/.actor/actor.json
index 944ceeb3d..7f7835da4 100644
--- a/test/e2e/automatic-persist-value/actor/apify.json
+++ b/test/e2e/automatic-persist-value/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-automatic-persist-value",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/actor/apify.json b/test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.json
similarity index 80%
rename from test/e2e/autoscaling-max-tasks-per-minute/actor/apify.json
rename to test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.json
index b6f64269e..e6e0c1ed5 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/actor/apify.json
+++ b/test/e2e/autoscaling-max-tasks-per-minute/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-autoscaling-max-tasks-per-minute",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/autoscaling-max-tasks-per-minute/test.mjs b/test/e2e/autoscaling-max-tasks-per-minute/test.mjs
index 6e9351c55..3979c69e0 100644
--- a/test/e2e/autoscaling-max-tasks-per-minute/test.mjs
+++ b/test/e2e/autoscaling-max-tasks-per-minute/test.mjs
@@ -7,6 +7,6 @@ const { stats } = await runActor(testActorDirname);
await expect(stats.requestsFinished === 2, 'All requests finished');
await expect(
- stats.crawlerRuntimeMillis > 60_000 && stats.crawlerRuntimeMillis < 65_000,
+ stats.crawlerRuntimeMillis > 60_000 && stats.crawlerRuntimeMillis < 70_000,
`Ran one task per minute, took ~1 minute to complete, but no more than that (${stats.crawlerRuntimeMillis}ms)`,
);
diff --git a/test/e2e/cheerio-default-ts/actor/apify.json b/test/e2e/cheerio-default-ts/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/cheerio-default-ts/actor/apify.json
rename to test/e2e/cheerio-default-ts/actor/.actor/actor.json
index d3e05a49e..d1802658a 100644
--- a/test/e2e/cheerio-default-ts/actor/apify.json
+++ b/test/e2e/cheerio-default-ts/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-default-ts",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-default/actor/apify.json b/test/e2e/cheerio-default/actor/.actor/actor.json
similarity index 77%
rename from test/e2e/cheerio-default/actor/apify.json
rename to test/e2e/cheerio-default/actor/.actor/actor.json
index 4d5d0a673..69e8b83b6 100644
--- a/test/e2e/cheerio-default/actor/apify.json
+++ b/test/e2e/cheerio-default/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-default",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-enqueue-links/actor/apify.json b/test/e2e/cheerio-enqueue-links/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/cheerio-enqueue-links/actor/apify.json
rename to test/e2e/cheerio-enqueue-links/actor/.actor/actor.json
index 8a288a3b4..d51768db2 100644
--- a/test/e2e/cheerio-enqueue-links/actor/apify.json
+++ b/test/e2e/cheerio-enqueue-links/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-enqueue-links",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-ignore-ssl-errors/actor/apify.json b/test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/cheerio-ignore-ssl-errors/actor/apify.json
rename to test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.json
index 3faacd490..fbf791bef 100644
--- a/test/e2e/cheerio-ignore-ssl-errors/actor/apify.json
+++ b/test/e2e/cheerio-ignore-ssl-errors/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-ignore-ssl-errors",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-initial-cookies/actor/apify.json b/test/e2e/cheerio-initial-cookies/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/cheerio-initial-cookies/actor/apify.json
rename to test/e2e/cheerio-initial-cookies/actor/.actor/actor.json
index 98ab258fe..e3df9a5b5 100644
--- a/test/e2e/cheerio-initial-cookies/actor/apify.json
+++ b/test/e2e/cheerio-initial-cookies/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-initial-cookies",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-max-requests/actor/apify.json b/test/e2e/cheerio-max-requests/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/cheerio-max-requests/actor/apify.json
rename to test/e2e/cheerio-max-requests/actor/.actor/actor.json
index 965ffae6d..28134cf02 100644
--- a/test/e2e/cheerio-max-requests/actor/apify.json
+++ b/test/e2e/cheerio-max-requests/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-max-requests",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-max-requests/actor/main.js b/test/e2e/cheerio-max-requests/actor/main.js
index bbd6cca41..3e1344ecc 100644
--- a/test/e2e/cheerio-max-requests/actor/main.js
+++ b/test/e2e/cheerio-max-requests/actor/main.js
@@ -15,9 +15,9 @@ await Actor.main(async () => {
const { url, userData: { label } } = request;
if (label === 'START') {
- const links = $('.ActorStoreItem').toArray().map((item) => $(item).attr('href'));
+ const links = $('a.card').toArray().map((item) => $(item).attr('href'));
for (const link of links) {
- const actorDetailUrl = `https://apify.com${link}`;
+ const actorDetailUrl = `https://crawlee.dev${link}`;
await crawler.addRequests([{
url: actorDetailUrl,
userData: { label: 'DETAIL' },
@@ -26,21 +26,19 @@ await Actor.main(async () => {
} else if (label === 'DETAIL') {
const uniqueIdentifier = url.split('/').slice(-2).join('/');
const title = $('header h1').text();
- const description = $('header span.actor-description').text();
- const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');
- const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');
+ const firstParagraph = $('header + p').text();
+ const modifiedDate = $('.theme-last-updated time').attr('datetime');
await Dataset.pushData({
url,
uniqueIdentifier,
title,
- description,
- modifiedDate: new Date(Number(modifiedDate)),
- runCount: Number(runCount),
+ firstParagraph,
+ modifiedDate,
});
}
},
});
- await crawler.run([{ url: 'https://apify.com/apify', userData: { label: 'START' } }]);
+ await crawler.run([{ url: 'https://crawlee.dev/docs/examples', userData: { label: 'START' } }]);
}, mainOptions);
diff --git a/test/e2e/cheerio-max-requests/test.mjs b/test/e2e/cheerio-max-requests/test.mjs
index fd8394e08..cf70bbf93 100644
--- a/test/e2e/cheerio-max-requests/test.mjs
+++ b/test/e2e/cheerio-max-requests/test.mjs
@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname);
await expect(stats.requestsFinished > 10, 'All requests finished');
await expect(datasetItems.length > 5 && datasetItems.length < 15, 'Number of dataset items');
await expect(
- validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),
+ validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'firstParagraph', 'modifiedDate']),
'Dataset items validation',
);
diff --git a/test/e2e/cheerio-page-info/actor/apify.json b/test/e2e/cheerio-page-info/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/cheerio-page-info/actor/apify.json
rename to test/e2e/cheerio-page-info/actor/.actor/actor.json
index d8aa10fb9..448dc8868 100644
--- a/test/e2e/cheerio-page-info/actor/apify.json
+++ b/test/e2e/cheerio-page-info/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-page-info",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/cheerio-page-info/actor/main.js b/test/e2e/cheerio-page-info/actor/main.js
index 62f6e4c4e..191217c29 100644
--- a/test/e2e/cheerio-page-info/actor/main.js
+++ b/test/e2e/cheerio-page-info/actor/main.js
@@ -12,7 +12,7 @@ const router = createCheerioRouter();
router.addHandler('START', async ({ enqueueLinks }) => {
await enqueueLinks({
label: 'DETAIL',
- globs: ['https://apify.com/apify/web-scraper'],
+ globs: ['**/examples/accept-user-input'],
});
});
@@ -21,17 +21,15 @@ router.addHandler('DETAIL', async ({ request, $ }) => {
const uniqueIdentifier = url.split('/').slice(-2).join('/');
const title = $('header h1').text();
- const description = $('header span.actor-description').text();
- const modifiedDate = $('ul.ActorHeader-stats time').attr('datetime');
- const runCount = $('ul.ActorHeader-stats > li:nth-of-type(3)').text().match(/[\d,]+/)[0].replace(/,/g, '');
+ const firstParagraph = $('header + p').text();
+ const modifiedDate = $('.theme-last-updated time').attr('datetime');
await Dataset.pushData({
url,
uniqueIdentifier,
title,
- description,
- modifiedDate: new Date(Number(modifiedDate)),
- runCount: Number(runCount),
+ firstParagraph,
+ modifiedDate,
});
});
@@ -40,5 +38,5 @@ await Actor.main(async () => {
requestHandler: router,
});
- await crawler.run([{ url: 'https://apify.com/apify', userData: { label: 'START' } }]);
+ await crawler.run([{ url: 'https://crawlee.dev/docs/3.0/examples', userData: { label: 'START' } }]);
}, mainOptions);
diff --git a/test/e2e/cheerio-page-info/test.mjs b/test/e2e/cheerio-page-info/test.mjs
index e051e3b84..6ed16a4f6 100644
--- a/test/e2e/cheerio-page-info/test.mjs
+++ b/test/e2e/cheerio-page-info/test.mjs
@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname);
await expect(stats.requestsFinished === 2, 'All requests finished');
await expect(datasetItems.length === 1, 'Number of dataset items');
await expect(
- validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),
+ validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'firstParagraph', 'modifiedDate']),
'Dataset items validation',
);
diff --git a/test/e2e/cheerio-throw-on-ssl-errors/actor/apify.json b/test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.json
similarity index 80%
rename from test/e2e/cheerio-throw-on-ssl-errors/actor/apify.json
rename to test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.json
index 71bf83458..051de1117 100644
--- a/test/e2e/cheerio-throw-on-ssl-errors/actor/apify.json
+++ b/test/e2e/cheerio-throw-on-ssl-errors/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-cheerio-throw-on-ssl-errors",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/input-json5/actor/apify.json b/test/e2e/input-json5/actor/.actor/actor.json
similarity index 77%
rename from test/e2e/input-json5/actor/apify.json
rename to test/e2e/input-json5/actor/.actor/actor.json
index aa48ca695..c3a5b0ca6 100644
--- a/test/e2e/input-json5/actor/apify.json
+++ b/test/e2e/input-json5/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-input-json5",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/jsdom-default-ts/actor/apify.json b/test/e2e/jsdom-default-ts/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/jsdom-default-ts/actor/apify.json
rename to test/e2e/jsdom-default-ts/actor/.actor/actor.json
index dfa4a1d45..976cd31c4 100644
--- a/test/e2e/jsdom-default-ts/actor/apify.json
+++ b/test/e2e/jsdom-default-ts/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-jsdom-default-ts",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/jsdom-react-ts/actor/apify.json b/test/e2e/jsdom-react-ts/actor/.actor/actor.json
similarity index 77%
rename from test/e2e/jsdom-react-ts/actor/apify.json
rename to test/e2e/jsdom-react-ts/actor/.actor/actor.json
index 834a0018b..9fe495397 100644
--- a/test/e2e/jsdom-react-ts/actor/apify.json
+++ b/test/e2e/jsdom-react-ts/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-jsdom-react-ts",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/migration/actor/apify.json b/test/e2e/migration/actor/.actor/actor.json
similarity index 76%
rename from test/e2e/migration/actor/apify.json
rename to test/e2e/migration/actor/.actor/actor.json
index f65813559..29134d82e 100644
--- a/test/e2e/migration/actor/apify.json
+++ b/test/e2e/migration/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-migration",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/migration/actor/main.js b/test/e2e/migration/actor/main.js
index 9aa1963f1..f06fc8ece 100644
--- a/test/e2e/migration/actor/main.js
+++ b/test/e2e/migration/actor/main.js
@@ -37,7 +37,7 @@ if (workerData !== '#actor') {
maxRequestsPerCrawl: 5,
async requestHandler({ enqueueLinks, request }) {
const { url } = request;
- await enqueueLinks({ pseudoUrls: ['https://apify.com[(/[\\w-]+)?]'] });
+ await enqueueLinks();
await Dataset.pushData({ url });
@@ -48,6 +48,6 @@ if (workerData !== '#actor') {
// eslint-disable-next-line no-underscore-dangle
Configuration.getGlobalConfig().getStorageClient().__purged = false;
- await crawler.run(['https://apify.com']);
+ await crawler.run(['https://crawlee.dev']);
}, mainOptions);
}
diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/apify.json b/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json
similarity index 82%
rename from test/e2e/playwright-chromium-experimental-containers/actor/apify.json
rename to test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json
index 7f43847ae..0be68bf20 100644
--- a/test/e2e/playwright-chromium-experimental-containers/actor/apify.json
+++ b/test/e2e/playwright-chromium-experimental-containers/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-playwright-chromium-experimental-containers",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
index de06663ad..58d737169 100644
--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/playwright-default/actor/apify.json b/test/e2e/playwright-default/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/playwright-default/actor/apify.json
rename to test/e2e/playwright-default/actor/.actor/actor.json
index ddddf0cba..9ed1dfad0 100644
--- a/test/e2e/playwright-default/actor/apify.json
+++ b/test/e2e/playwright-default/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-playwright-default",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfile
index de06663ad..58d737169 100644
--- a/test/e2e/playwright-default/actor/Dockerfile
+++ b/test/e2e/playwright-default/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/playwright-default/actor/main.js b/test/e2e/playwright-default/actor/main.js
index 5592a895b..5a725fbad 100644
--- a/test/e2e/playwright-default/actor/main.js
+++ b/test/e2e/playwright-default/actor/main.js
@@ -16,9 +16,11 @@ await Actor.main(async () => {
const { url } = request;
const pageTitle = await page.title();
await Dataset.pushData({ url, pageTitle });
- await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });
+ await enqueueLinks({
+ globs: ['**/3.0/examples/*'],
+ });
},
});
- await crawler.run(['https://apify.com']);
+ await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);
}, mainOptions);
diff --git a/test/e2e/playwright-default/test.mjs b/test/e2e/playwright-default/test.mjs
index 7b0d5153c..b82824ae1 100644
--- a/test/e2e/playwright-default/test.mjs
+++ b/test/e2e/playwright-default/test.mjs
@@ -5,6 +5,6 @@ await initialize(testActorDirname);
const { stats, datasetItems } = await runActor(testActorDirname, 16384);
-await expect(stats.requestsFinished > 50, 'All requests finished');
-await expect(datasetItems.length > 50 && datasetItems.length < 150, 'Number of dataset items');
+await expect(stats.requestsFinished > 15, 'All requests finished');
+await expect(datasetItems.length > 15 && datasetItems.length < 25, 'Number of dataset items');
await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');
diff --git a/test/e2e/playwright-enqueue-links/actor/apify.json b/test/e2e/playwright-enqueue-links/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/playwright-enqueue-links/actor/apify.json
rename to test/e2e/playwright-enqueue-links/actor/.actor/actor.json
index e36af37c1..7294db34b 100644
--- a/test/e2e/playwright-enqueue-links/actor/apify.json
+++ b/test/e2e/playwright-enqueue-links/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-playwright-enqueue-links",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfile
index de06663ad..58d737169 100644
--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile
+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/apify.json b/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json
similarity index 82%
rename from test/e2e/playwright-firefox-experimental-containers/actor/apify.json
rename to test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json
index 8bc7a689e..d1bf754a5 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/apify.json
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-playwright-firefox-experimental-containers",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
index 8a1c0fff0..af65b94eb 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/playwright-initial-cookies/actor/apify.json b/test/e2e/playwright-initial-cookies/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/playwright-initial-cookies/actor/apify.json
rename to test/e2e/playwright-initial-cookies/actor/.actor/actor.json
index ca72b5fff..0404c8af3 100644
--- a/test/e2e/playwright-initial-cookies/actor/apify.json
+++ b/test/e2e/playwright-initial-cookies/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-playwright-initial-cookies",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile
index de06663ad..58d737169 100644
--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile
+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/proxy-rotation/actor/apify.json b/test/e2e/proxy-rotation/actor/.actor/actor.json
similarity index 77%
rename from test/e2e/proxy-rotation/actor/apify.json
rename to test/e2e/proxy-rotation/actor/.actor/actor.json
index eb8ac5e01..4af4d0489 100644
--- a/test/e2e/proxy-rotation/actor/apify.json
+++ b/test/e2e/proxy-rotation/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-proxy-rotation",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/proxy-rotation/actor/Dockerfile
+++ b/test/e2e/proxy-rotation/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-default/actor/apify.json b/test/e2e/puppeteer-default/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/puppeteer-default/actor/apify.json
rename to test/e2e/puppeteer-default/actor/.actor/actor.json
index f57a60ea8..f2fa978c5 100644
--- a/test/e2e/puppeteer-default/actor/apify.json
+++ b/test/e2e/puppeteer-default/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-default",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-default/actor/Dockerfile
+++ b/test/e2e/puppeteer-default/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.js
index 291249bda..699239d1a 100644
--- a/test/e2e/puppeteer-default/actor/main.js
+++ b/test/e2e/puppeteer-default/actor/main.js
@@ -16,9 +16,11 @@ await Actor.main(async () => {
const { url } = request;
const pageTitle = await page.title();
await Dataset.pushData({ url, pageTitle });
- await enqueueLinks({ regexps: [/^https:\/\/apify\.com(\/[\w-]+)?$/i] });
+ await enqueueLinks({
+ globs: ['**/3.0/examples/*'],
+ });
},
});
- await crawler.run(['https://apify.com']);
+ await crawler.run(['https://crawlee.dev/docs/3.0/examples/']);
}, mainOptions);
diff --git a/test/e2e/puppeteer-default/test.mjs b/test/e2e/puppeteer-default/test.mjs
index 7b0d5153c..b82824ae1 100644
--- a/test/e2e/puppeteer-default/test.mjs
+++ b/test/e2e/puppeteer-default/test.mjs
@@ -5,6 +5,6 @@ await initialize(testActorDirname);
const { stats, datasetItems } = await runActor(testActorDirname, 16384);
-await expect(stats.requestsFinished > 50, 'All requests finished');
-await expect(datasetItems.length > 50 && datasetItems.length < 150, 'Number of dataset items');
+await expect(stats.requestsFinished > 15, 'All requests finished');
+await expect(datasetItems.length > 15 && datasetItems.length < 25, 'Number of dataset items');
await expect(validateDataset(datasetItems, ['url', 'pageTitle']), 'Dataset items validation');
diff --git a/test/e2e/puppeteer-enqueue-links/actor/apify.json b/test/e2e/puppeteer-enqueue-links/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/puppeteer-enqueue-links/actor/apify.json
rename to test/e2e/puppeteer-enqueue-links/actor/.actor/actor.json
index 7dd9d68c5..7beecb430 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/apify.json
+++ b/test/e2e/puppeteer-enqueue-links/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-enqueue-links",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/apify.json b/test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.json
similarity index 80%
rename from test/e2e/puppeteer-ignore-ssl-errors/actor/apify.json
rename to test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.json
index 049cfe03d..abbeda5d9 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/apify.json
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-ignore-ssl-errors",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-initial-cookies/actor/apify.json b/test/e2e/puppeteer-initial-cookies/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/puppeteer-initial-cookies/actor/apify.json
rename to test/e2e/puppeteer-initial-cookies/actor/.actor/actor.json
index aa2946018..328c5bb8a 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/apify.json
+++ b/test/e2e/puppeteer-initial-cookies/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-initial-cookies",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-page-info/actor/apify.json b/test/e2e/puppeteer-page-info/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/puppeteer-page-info/actor/apify.json
rename to test/e2e/puppeteer-page-info/actor/.actor/actor.json
index e136f456e..285cbce73 100644
--- a/test/e2e/puppeteer-page-info/actor/apify.json
+++ b/test/e2e/puppeteer-page-info/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-page-info",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-page-info/actor/Dockerfile
+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-page-info/actor/main.js b/test/e2e/puppeteer-page-info/actor/main.js
index ee853ff0e..e851d4569 100644
--- a/test/e2e/puppeteer-page-info/actor/main.js
+++ b/test/e2e/puppeteer-page-info/actor/main.js
@@ -17,7 +17,7 @@ await Actor.main(async () => {
if (label === 'START') {
await enqueueLinks({
- globs: [{ glob: 'https://apify.com/apify/web-scraper', userData: { label: 'DETAIL' } }],
+ globs: ['**/examples/accept-user-input'], userData: { label: 'DETAIL' },
});
}
@@ -27,28 +27,22 @@ await Actor.main(async () => {
const uniqueIdentifier = url.split('/').slice(-2).join('/');
const titleP = page.$eval('header h1', ((el) => el.textContent));
- const descriptionP = page.$eval('header span.actor-description', ((el) => el.textContent));
- const modifiedTimestampP = page.$eval('ul.ActorHeader-stats time', (el) => el.getAttribute('datetime'));
- const runCountTextP = page.$eval('ul.ActorHeader-stats > li:nth-of-type(3)', ((el) => el.textContent));
+ const firstParagraphP = page.$eval('header + p', ((el) => el.textContent));
+ const modifiedDateP = page.$eval('.theme-last-updated time', (el) => el.getAttribute('datetime'));
const [
title,
description,
- modifiedTimestamp,
- runCountText,
+ modifiedDate,
] = await Promise.all([
titleP,
- descriptionP,
- modifiedTimestampP,
- runCountTextP,
+ firstParagraphP,
+ modifiedDateP,
]);
- const modifiedDate = new Date(Number(modifiedTimestamp));
- const runCount = Number(runCountText.match(/[\d,]+/)[0].replace(/,/g, ''));
-
- await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate, runCount });
+ await Dataset.pushData({ url, uniqueIdentifier, title, description, modifiedDate });
}
},
});
- await crawler.run([{ url: 'https://apify.com/store', userData: { label: 'START' } }]);
+ await crawler.run([{ url: 'https://crawlee.dev/docs/3.0/examples', userData: { label: 'START' } }]);
}, mainOptions);
diff --git a/test/e2e/puppeteer-page-info/test.mjs b/test/e2e/puppeteer-page-info/test.mjs
index 3f9f005d1..06d47068c 100644
--- a/test/e2e/puppeteer-page-info/test.mjs
+++ b/test/e2e/puppeteer-page-info/test.mjs
@@ -8,6 +8,6 @@ const { stats, datasetItems } = await runActor(testActorDirname, 16384);
await expect(stats.requestsFinished === 2, 'All requests finished');
await expect(datasetItems.length === 1, 'Number of dataset items');
await expect(
- validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate', 'runCount']),
+ validateDataset(datasetItems, ['url', 'title', 'uniqueIdentifier', 'description', 'modifiedDate']),
'Dataset items validation',
);
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/apify.json b/test/e2e/puppeteer-store-pagination-jquery/actor/.actor/actor.json
similarity index 80%
rename from test/e2e/puppeteer-store-pagination-jquery/actor/apify.json
rename to test/e2e/puppeteer-store-pagination-jquery/actor/.actor/actor.json
index 238376e78..8d6941f71 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/apify.json
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-store-pagination-jquery",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
index f18f8b5dc..8d641cd52 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/main.js
@@ -26,7 +26,7 @@ await Actor.main(async () => {
await page.waitForNetworkIdle();
// Enqueue all loaded links
await enqueueLinks({
- selector: 'a.ActorStoreItem',
+ selector: '[data-test="actorCard"] > a',
globs: [{ glob: 'https://apify.com/*/*', userData: { label: 'DETAIL' } }],
});
log.info(`Enqueued actors for page ${pageNo}`);
diff --git a/test/e2e/puppeteer-store-pagination/actor/apify.json b/test/e2e/puppeteer-store-pagination/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/puppeteer-store-pagination/actor/apify.json
rename to test/e2e/puppeteer-store-pagination/actor/.actor/actor.json
index d87f9cd56..0fd41a53d 100644
--- a/test/e2e/puppeteer-store-pagination/actor/apify.json
+++ b/test/e2e/puppeteer-store-pagination/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-store-pagination",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/puppeteer-store-pagination/actor/main.js b/test/e2e/puppeteer-store-pagination/actor/main.js
index 9357d65d7..86726517a 100644
--- a/test/e2e/puppeteer-store-pagination/actor/main.js
+++ b/test/e2e/puppeteer-store-pagination/actor/main.js
@@ -21,7 +21,7 @@ crawler.router.addHandler('START', async ({ log, enqueueLinks, page }) => {
await page.waitForNetworkIdle();
// Enqueue all loaded links
await enqueueLinks({
- selector: 'a.ActorStoreItem',
+ selector: '[data-test="actorCard"] > a',
label: 'DETAIL',
globs: ['https://apify.com/*/*'],
});
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/apify.json b/test/e2e/puppeteer-throw-on-ssl-errors/actor/.actor/actor.json
similarity index 80%
rename from test/e2e/puppeteer-throw-on-ssl-errors/actor/apify.json
rename to test/e2e/puppeteer-throw-on-ssl-errors/actor/.actor/actor.json
index 8c375a749..224de98d2 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/apify.json
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-puppeteer-throw-on-ssl-errors",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
index 7c7d30b82..de6b194d0 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
@@ -12,7 +12,7 @@ RUN rm -r node_modules
COPY --from=builder /node_modules ./node_modules
COPY --from=builder /packages ./packages
COPY --from=builder /package*.json ./
-COPY /apify.json ./
+COPY /.actor ./.actor
COPY /main.js ./
RUN echo "Installed NPM packages:" \
diff --git a/test/e2e/request-queue-zero-concurrency/actor/apify.json b/test/e2e/request-queue-zero-concurrency/actor/.actor/actor.json
similarity index 80%
rename from test/e2e/request-queue-zero-concurrency/actor/apify.json
rename to test/e2e/request-queue-zero-concurrency/actor/.actor/actor.json
index ccfd2bacd..959ec7eaa 100644
--- a/test/e2e/request-queue-zero-concurrency/actor/apify.json
+++ b/test/e2e/request-queue-zero-concurrency/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-request-queue-zero-concurrency",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/request-skip-navigation/actor/apify.json b/test/e2e/request-skip-navigation/actor/.actor/actor.json
similarity index 79%
rename from test/e2e/request-skip-navigation/actor/apify.json
rename to test/e2e/request-skip-navigation/actor/.actor/actor.json
index 391f5a195..0f06a7057 100644
--- a/test/e2e/request-skip-navigation/actor/apify.json
+++ b/test/e2e/request-skip-navigation/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-request-skip-navigation",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/session-rotation/actor/apify.json b/test/e2e/session-rotation/actor/.actor/actor.json
similarity index 78%
rename from test/e2e/session-rotation/actor/apify.json
rename to test/e2e/session-rotation/actor/.actor/actor.json
index bac48c79c..7308531ed 100644
--- a/test/e2e/session-rotation/actor/apify.json
+++ b/test/e2e/session-rotation/actor/.actor/actor.json
@@ -1,4 +1,5 @@
{
+ "actorSpecification": 1,
"name": "test-session-rotation",
"version": "0.0",
"buildTag": "latest",
diff --git a/test/e2e/session-rotation/actor/main.js b/test/e2e/session-rotation/actor/main.js
index 06a711315..107dd47c0 100644
--- a/test/e2e/session-rotation/actor/main.js
+++ b/test/e2e/session-rotation/actor/main.js
@@ -17,7 +17,7 @@ await Actor.main(async () => {
},
requestHandler: async ({ session }) => {
const { id, usageCount, errorScore } = session;
- Actor.pushData({ id, usageCount, errorScore });
+ await Actor.pushData({ id, usageCount, errorScore });
throw new Error('retry');
},
});
diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjs
index 1603168b9..85f9aa29f 100644
--- a/test/e2e/tools.mjs
+++ b/test/e2e/tools.mjs
@@ -4,13 +4,17 @@ import { existsSync } from 'node:fs';
import { readdir, readFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { setTimeout } from 'node:timers/promises';
-import { execSync } from 'node:child_process';
+import { execSync as execSyncOriginal } from 'node:child_process';
import { got } from 'got';
import fs from 'fs-extra';
import { Actor } from 'apify';
// eslint-disable-next-line import/no-relative-packages
import { URL_NO_COMMAS_REGEX } from '../../packages/utils/dist/index.mjs';
+function execSync(command, options) {
+ return execSyncOriginal(command, { ...options, encoding: 'utf-8' });
+}
+
/**
* @param {string} name
*/