Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(e2e): stream worker logs live and show test-start marker (#3675)
Martin Adámek8374db825d6d
diff --git a/test/e2e/cheerio-enqueue-links-base/test.mjs b/test/e2e/cheerio-enqueue-links-base/test.mjsindex 1af2a7738..448973a3e 100644--- a/test/e2e/cheerio-enqueue-links-base/test.mjs+++ b/test/e2e/cheerio-enqueue-links-base/test.mjs@@ -1,4 +1,13 @@-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';+import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';++// The actor spins up an HTTP server on 127.0.0.1; that works inside the+// in-process LOCAL/MEMORY worker but is unreachable from the Apify platform+// container, so the run never finishes. Base-href handling is pure parsing+// logic in @crawlee/utils that doesn't depend on the storage backend, so+// LOCAL+MEMORY coverage is sufficient.+if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {+ await skipTest('localhost fixture is not reachable from the Apify platform');+}const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/playwright-enqueue-links-base/test.mjs b/test/e2e/playwright-enqueue-links-base/test.mjsindex 1af2a7738..448973a3e 100644--- a/test/e2e/playwright-enqueue-links-base/test.mjs+++ b/test/e2e/playwright-enqueue-links-base/test.mjs@@ -1,4 +1,13 @@-import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';+import { initialize, getActorTestDir, runActor, expect, skipTest } from '../tools.mjs';++// The actor spins up an HTTP server on 127.0.0.1; that works inside the+// in-process LOCAL/MEMORY worker but is unreachable from the Apify platform+// container, so the run never finishes. Base-href handling is pure parsing+// logic in @crawlee/utils that doesn't depend on the storage backend, so+// LOCAL+MEMORY coverage is sufficient.+if (process.env.STORAGE_IMPLEMENTATION === 'PLATFORM') {+ await skipTest('localhost fixture is not reachable from the Apify platform');+}const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/run.mjs b/test/e2e/run.mjsindex aef291b7a..a331e945d 100644--- a/test/e2e/run.mjs+++ b/test/e2e/run.mjs@@ -46,62 +46,56 @@ async function run() {}const now = Date.now();+ console.log(`${colors.yellow(`[${dir.name}] `)}${colors.grey('Test starting...')}`);const worker = new Worker(fileURLToPath(import.meta.url), {workerData: dir.name,stdout: true,stderr: true,});let seenFirst = false;- /** @type Map<string, string[]> */- const allLogs = new Map();- worker.stderr.on('data', (data) => {- const str = data.toString();- const taskLogs = allLogs.get(dir.name) ?? [];- allLogs.set(dir.name, taskLogs);- taskLogs.push(str);- });- worker.stdout.on('data', (data) => {- const str = data.toString();- const taskLogs = allLogs.get(dir.name) ?? [];- allLogs.set(dir.name, taskLogs);- taskLogs.push(str);-- if (str.startsWith('[test skipped]')) {- return;- }-- if (str.startsWith('[init]')) {- seenFirst = true;- return;- }-- if (!seenFirst) {- console.log(- `${colors.red('[fatal]')} test ${colors.yellow(- `[${dir.name}]`,- )} did not call "initialize(import.meta.url)"!`,- );- worker.terminate();- return;- }-- if (- process.env.STORAGE_IMPLEMENTATION === 'PLATFORM' &&- (str.startsWith('[build]') || str.startsWith('[run]') || str.startsWith('[kv]'))- ) {- const platformStatsMessage = str.match(/\[(?:run|build|kv)] (.*)/);- if (platformStatsMessage) {- console.log(`${colors.yellow(`[${dir.name}] `)}${colors.grey(platformStatsMessage[1])}`);+ const prefix = colors.yellow(`[${dir.name}] `);+ // Surface only lines that start with a structured `[…]` marker (init,+ // assertion, build, run, kv, test skipped, etc.). Everything else+ // (crawler INFO logs, per-URL request handler logs, npm warnings, …)+ // is noise on a green run; buffer it and re-emit from the exit+ // handler iff the test failed.+ const deferredOnSuccess = [];++ // Line-buffered streaming so prefixed lines stay intact across chunk boundaries.+ const streamLines = (stream, sink) => {+ let buffer = '';+ stream.on('data', (chunk) => {+ buffer += chunk.toString();+ const lines = buffer.split('\n');+ buffer = lines.pop();+ for (const line of lines) {+ if (line === '') continue;++ if (!line.startsWith('[')) {+ if (!seenFirst) {+ console.log(+ `${colors.red('[fatal]')} test ${colors.yellow(+ `[${dir.name}]`,+ )} did not call "initialize(import.meta.url)"!`,+ );+ worker.terminate();+ return;+ }+ deferredOnSuccess.push(line);+ continue;+ }++ seenFirst = true;+ sink(`${prefix}${line}`);}- }-- const match = str.match(/\[assertion] (passed|failed): (.*)/);+ });+ stream.on('end', () => {+ if (buffer !== '') sink(`${prefix}${buffer}`);+ });+ };- if (match) {- const c = match[1] === 'passed' ? colors.green : colors.red;- console.log(`${colors.yellow(`[${dir.name}] `)}${match[2]}: ${c(match[1])}`);- }- });+ streamLines(worker.stdout, (line) => console.log(line));+ streamLines(worker.stderr, (line) => console.error(line));worker.on('error', (err) => {// If the worker emits any error, we want to exit with a non-zero code@@ -111,10 +105,14 @@ async function run() {const exitHandler = async (code) => {if (code === SKIPPED_TEST_CLOSE_CODE) {- console.log(`Test ${colors.yellow(`[${dir.name}]`)} was skipped`);+ console.log(`${prefix}${colors.grey('Test skipped')}`);return;}+ if (code !== 0) {+ for (const line of deferredOnSuccess) console.log(`${prefix}${line}`);+ }+const took = (Date.now() - now) / 1000;const status = code === 0 ? 'success' : 'failure';const color = code === 0 ? 'green' : 'red';@@ -132,12 +130,6 @@ async function run() {await clearPackages(`${basePath}/${dir.name}`);}- const taskLogs = allLogs.get(dir.name);-- if (code !== 0 && taskLogs?.length > 0) {- console.log(taskLogs.join('\n'));- }-if (status === 'failure') failure = true;};diff --git a/test/e2e/tools.mjs b/test/e2e/tools.mjsindex 9e2aadfe4..6274cc759 100644--- a/test/e2e/tools.mjs+++ b/test/e2e/tools.mjs@@ -71,9 +71,16 @@ export function getActorTestDir(url) {export async function pushActor(client, dirName) {await copyPackages(dirName);try {+ // Capture all stdio (default would inherit stderr and flood the runner+ // with the platform's Docker/npm build output on every test). The+ // catch block below re-prints stdout/stderr if the push itself fails,+ // so failure diagnostics stay intact. maxBuffer bumped because the+ // build log can be tens of MB.execSync('apify push', {cwd: dirName,env: { ...process.env, GIT_CEILING_DIRECTORIES: dirname(dirName) },+ stdio: 'pipe',+ maxBuffer: 50 * 1024 * 1024,});} catch (err) {console.error(colors.red(`Failed to push actor to the Apify platform. (signal ${colors.yellow(err.signal)})`));