Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
/*
Copyright (c) Microsoft Corporation.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import type { TestAttachment, TestCase, TestCaseSummary, TestResult, TestStep } from './types';
import * as React from 'react';
import { TreeItem } from './treeItem';
import { formatUrl } from './utils';
import { msToString } from '@isomorphic/formatUtils';
import { AutoChip } from './chip';
import { traceImage } from './images';
import { Anchor, AttachmentLink, generateTraceUrl, testResultHref, useSearchParams } from './links';
import { statusIcon } from './statusIcon';
import type { ImageDiff } from '@web/shared/imageDiffView';
import { ImageDiffView } from '@web/shared/imageDiffView';
import { CodeSnippet, PromptButton, TestScreenshotErrorView } from './testErrorView';
import * as icons from './icons';
import './testResultView.css';
import { useAsyncMemo } from '@web/uiUtils';
import type { LoadedReport } from './loadedReport';
import { TestCaseListView } from './testFileView';
import { stripAnsiEscapes } from '@isomorphic/stringUtils';
interface ImageDiffWithAnchors extends ImageDiff {
anchors: string[];
}
function groupImageDiffs(screenshots: Set<TestAttachment>, result: TestResult): ImageDiffWithAnchors[] {
const snapshotNameToImageDiff = new Map<string, ImageDiffWithAnchors>();
for (const attachment of screenshots) {
const match = attachment.name.match(/^(.*)-(expected|actual|diff|previous)(\.[^.]+)?$/);
if (!match)
continue;
const [, name, category, extension = ''] = match;
const snapshotName = name + extension;
let imageDiff = snapshotNameToImageDiff.get(snapshotName);
if (!imageDiff) {
imageDiff = { name: snapshotName, anchors: [`attachment-${name}`] };
snapshotNameToImageDiff.set(snapshotName, imageDiff);
}
imageDiff.anchors.push(`attachment-${result.attachments.indexOf(attachment)}`);
if (category === 'actual')
imageDiff.actual = { attachment };
if (category === 'expected')
imageDiff.expected = { attachment, title: 'Expected' };
if (category === 'previous')
imageDiff.expected = { attachment, title: 'Previous' };
if (category === 'diff')
imageDiff.diff = { attachment };
}
for (const [name, diff] of snapshotNameToImageDiff) {
if (!diff.actual || !diff.expected) {
snapshotNameToImageDiff.delete(name);
} else {
screenshots.delete(diff.actual.attachment);
screenshots.delete(diff.expected.attachment);
screenshots.delete(diff.diff?.attachment!);
}
}
return [...snapshotNameToImageDiff.values()];
}
export const TestResultView: React.FC<{
report: LoadedReport,
test: TestCase,
result: TestResult,
}> = ({ report, test, result }) => {
const { screenshots, videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors, errorContext } = React.useMemo(() => {
const attachments = result.attachments.filter(a => !a.name.startsWith('_'));
const screenshots = new Set(attachments.filter(a => a.contentType.startsWith('image/')));
const screenshotAnchors = [...screenshots].map(a => `attachment-${attachments.indexOf(a)}`);
const videos = attachments.filter(a => a.contentType.startsWith('video/'));
const traces = attachments.filter(a => a.name === 'trace');
const errorContext = attachments.find(a => a.name === 'error-context');
const otherAttachments = new Set<TestAttachment>(attachments);
[...screenshots, ...videos, ...traces].forEach(a => otherAttachments.delete(a));
const otherAttachmentAnchors = [...otherAttachments].map(a => `attachment-${attachments.indexOf(a)}`);
const diffs = groupImageDiffs(screenshots, result);
const errors = result.errors.map(e => e.message);
return { screenshots: [...screenshots], videos, traces, otherAttachments, diffs, errors, otherAttachmentAnchors, screenshotAnchors, errorContext };
}, [result]);
const [stepFilterText, setStepFilterText] = React.useState('');
React.useEffect(() => setStepFilterText(''), [result]);
const waterfall = React.useMemo(() => {
let startTime = Infinity;
let endTime = -Infinity;
const visit = (step: TestStep) => {
const start = new Date(step.startTime).valueOf();
startTime = Math.min(startTime, start);
endTime = Math.max(endTime, start + Math.max(0, step.duration));
step.steps.forEach(visit);
};
result.steps.forEach(visit);
return { startTime, duration: Math.max(1, endTime - startTime) };
}, [result]);
const prompt = useAsyncMemo(async () => {
if (report.json().options?.noCopyPrompt)
return undefined;
if (!errorContext)
return undefined;
let text = errorContext.path ? await fetch(errorContext.path).then(r => r.text()) : errorContext.body;
if (!text)
return undefined;
const stdoutAttachment = result.attachments.find(a => a.name === 'stdout');
const stderrAttachment = result.attachments.find(a => a.name === 'stderr');
const stdout = stdoutAttachment?.body && stdoutAttachment.contentType === 'text/plain' ? stdoutAttachment.body : undefined;
const stderr = stderrAttachment?.body && stderrAttachment.contentType === 'text/plain' ? stderrAttachment.body : undefined;
if (stdout)
text += '\n\n# Stdout\n\n```\n' + stripAnsiEscapes(stdout) + '\n```';
if (stderr)
text += '\n\n# Stderr\n\n```\n' + stripAnsiEscapes(stderr) + '\n```';
const metadata = report.json().metadata;
if (metadata?.gitDiff)
text += '\n\n# Local changes\n\n```diff\n' + metadata.gitDiff + '\n```';
return text;
}, [errorContext, report, result], undefined);
return <div className='test-result'>
{!!errors.length && <AutoChip header='Errors'>
{prompt && (
<div style={{ position: 'absolute', right: '16px', padding: '10px', zIndex: 1 }}>
<PromptButton prompt={prompt} />
</div>
)}
{errors.map((error, index) => {
const diff = pickDiffForError(error, diffs);
return <>
<CodeSnippet key={'test-result-error-message-' + index} code={error}/>
{diff && <TestScreenshotErrorView diff={diff}></TestScreenshotErrorView>}
</>;
})}
</AutoChip>}
{!!result.steps.length && <AutoChip header='Test Steps'>
<form className='subnav-search step-filter' onSubmit={e => e.preventDefault()}>
{icons.search()}
<input className='form-control subnav-search-input input-contrast width-full' type='search' spellCheck={false} placeholder='Filter steps' aria-label='Filter steps' value={stepFilterText} onChange={e => setStepFilterText(e.target.value)} />
</form>
{result.steps.map((step, i) => <StepTreeItem key={`step-${i}`} step={step} result={result} test={test} depth={0} filterText={stepFilterText} waterfall={waterfall}/>)}
</AutoChip>}
{diffs.map((diff, index) =>
<Anchor key={`diff-${index}`} id={diff.anchors}>
<AutoChip dataTestId='test-results-image-diff' header={`Image mismatch: ${diff.name}`} revealOnAnchorId={diff.anchors}>
<ImageDiffView diff={diff}/>
</AutoChip>
</Anchor>
)}
{!!screenshots.length && <AutoChip header='Screenshots' revealOnAnchorId={screenshotAnchors}>
{screenshots.map((a, i) => {
return <Anchor key={`screenshot-${i}`} id={`attachment-${result.attachments.indexOf(a)}`}>
<a href={formatUrl(a.path)}>
<img className='screenshot' src={formatUrl(a.path)} />
</a>
<AttachmentLink attachment={a} result={result}></AttachmentLink>
</Anchor>;
})}
</AutoChip>}
{!!traces.length && <Anchor id='attachment-trace'><AutoChip header='Traces' revealOnAnchorId='attachment-trace'>
{traces.map((a, i) => <div key={`trace-${i}`}>
<a href={formatUrl(generateTraceUrl([a]))}>
<img className='screenshot' src={traceImage} style={{ width: 192, height: 117, marginLeft: 20 }} />
</a>
<AttachmentLink attachment={a} result={result} linkName={traces.length === 1 ? 'trace' : `trace-${i + 1}`}></AttachmentLink>
</div>)}
</AutoChip></Anchor>}
{!!videos.length && <Anchor id='attachment-video'><AutoChip header='Videos' revealOnAnchorId='attachment-video'>
{videos.map(a => <div key={a.path}>
<video controls>
<source src={formatUrl(a.path)} type={a.contentType}/>
</video>
<AttachmentLink attachment={a} result={result}></AttachmentLink>
</div>)}
</AutoChip></Anchor>}
{!!otherAttachments.size && <AutoChip header='Attachments' revealOnAnchorId={otherAttachmentAnchors} dataTestId='attachments'>
{[...otherAttachments].map((a, i) =>
<Anchor key={`attachment-link-${i}`} id={`attachment-${result.attachments.indexOf(a)}`}>
<AttachmentLink attachment={a} result={result} openInNewTab={a.contentType.startsWith('text/html')} />
</Anchor>
)}
</AutoChip>}
<AutoChip header={`Executed in Worker #${result.workerIndex}`} dataTestId='worker-test-list' initialExpanded={false} noInsets={true} body={() => {
const list = buildWorkerLists(report).get(result.workerIndex) || { tests: [], runs: [] };
return <TestCaseListView
tests={list.tests}
runs={list.runs}
projectNames={report.json().projectNames}
selectedTestId={test.testId}
/>;
}}/>
</div>;
};
function pickDiffForError(error: string, diffs: ImageDiff[]): ImageDiff | undefined {
const firstLine = error.split('\n')[0];
if (!firstLine.includes('toHaveScreenshot') && !firstLine.includes('toMatchSnapshot'))
return undefined;
return diffs.find(diff => error.includes(diff.name));
}
function stepMatchesFilter(step: TestStep, filterText: string): boolean {
const text = step.subtitle ? `${step.title} ${step.subtitle}` : step.title;
return text.toLowerCase().includes(filterText.toLowerCase());
}
function stepChildrenMatchFilter(step: TestStep, filterText: string): boolean {
return step.steps.some(s => stepMatchesFilter(s, filterText) || stepChildrenMatchFilter(s, filterText));
}
function highlightFilterText(text: string, filterText: string): React.ReactNode[] {
const unmatched = text.toLowerCase().split(filterText.toLowerCase());
const parts: React.ReactNode[] = [];
let index = 0;
for (let i = 0; i < unmatched.length; i++) {
if (i) {
parts.push(<span key={`highlight-${i}`} className='step-title-highlight'>{text.substring(index, index + filterText.length)}</span>);
index += filterText.length;
}
parts.push(text.substring(index, index + unmatched[i].length));
index += unmatched[i].length;
}
return parts;
}
function stepHasDescendantAttachments(step: TestStep): boolean {
return step.steps.some(s => s.attachments.length > 0 || stepHasDescendantAttachments(s));
}
function waterfallBlockStyle(step: TestStep, waterfall: { startTime: number, duration: number }): React.CSSProperties {
const startOffset = new Date(step.startTime).valueOf() - waterfall.startTime;
const left = Math.min(100, Math.max(0, startOffset / waterfall.duration * 100));
const width = Math.min(100 - left, Math.max(0, step.duration) / waterfall.duration * 100);
return { left: `${left}%`, width: `${width}%` };
}
const StepTreeItem: React.FC<{
test: TestCase;
result: TestResult;
step: TestStep;
depth: number,
filterText?: string,
waterfall: { startTime: number, duration: number },
}> = ({ test, step, result, depth, filterText, waterfall }) => {
const searchParams = useSearchParams();
let expandByDefault = false;
let title: React.ReactNode = <span>{step.title}</span>;
let subtitle: React.ReactNode = step.subtitle;
if (filterText) {
const matchesFilter = !!filterText && stepMatchesFilter(step, filterText);
const childrenMatchFilter = !!filterText && stepChildrenMatchFilter(step, filterText);
if (!matchesFilter && !childrenMatchFilter)
return null;
expandByDefault = childrenMatchFilter;
if (matchesFilter) {
title = highlightFilterText(step.title, filterText);
if (step.subtitle)
subtitle = highlightFilterText(step.subtitle, filterText);
}
}
return <TreeItem title={<div aria-label={step.subtitle ? `${step.title} ${step.subtitle}` : step.title} className='step-title-container'>
{statusIcon(step.error || step.duration === -1 ? 'failed' : (step.skipped ? 'skipped' : 'passed'))}
<span className='step-title-text'>
{title}
{step.subtitle && <span className='step-subtitle'> {subtitle}</span>}
{step.count > 1 && <> ✕ <span className='test-result-counter'>{step.count}</span></>}
{step.location && <span className='test-result-path'>— {step.location.file}:{step.location.line}</span>}
</span>
<span className='step-spacer'></span>
{step.attachments.length > 0 && <a
className='step-attachment-link'
title={`reveal attachment`}
href={formatUrl(testResultHref({ test, result, anchor: `attachment-${step.attachments[0]}` }, searchParams))}
onClick={evt => { evt.stopPropagation(); }}>
{icons.attachment()}
</a>}
{step.attachments.length === 0 && stepHasDescendantAttachments(step) && <span
className='step-indirect-attachment-indicator'
title='contains attachment'
aria-label='contains attachment'>
{icons.indirectAttachment()}
</span>}
<span className='step-waterfall'>
<span className='step-waterfall-block' style={waterfallBlockStyle(step, waterfall)}></span>
</span>
<span className='step-duration'>{msToString(step.duration)}</span>
</div>} loadChildren={step.steps.length || step.snippet ? () => {
const snippet = step.snippet ? [<CodeSnippet testId='test-snippet' key='line' code={step.snippet} />] : [];
const steps = step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1} result={result} test={test} filterText={filterText} waterfall={waterfall} />);
return snippet.concat(steps);
} : undefined} depth={depth} expandByDefault={expandByDefault}/>;
};
type WorkerLists = Map<number, { tests: TestCaseSummary[], runs: number[] }>;
const kWorkerListsSymbol = Symbol('workerLists');
function buildWorkerLists(report: LoadedReport): WorkerLists {
let data: WorkerLists | undefined = (report as any)[kWorkerListsSymbol];
if (!data) {
const lists = new Map<number, { test: TestCaseSummary, time: number, run: number }[]>();
for (const file of report.json().files) {
for (const test of file.tests) {
for (let index = 0; index < test.results.length; index++) {
let list = lists.get(test.results[index].workerIndex);
if (!list) {
list = [];
lists.set(test.results[index].workerIndex, list);
}
list.push({ test, time: new Date(test.results[index].startTime).valueOf(), run: index });
}
}
}
data = new Map();
for (const [workerIndex, list] of lists) {
list.sort((a, b) => a.time - b.time);
data.set(workerIndex, { tests: list.map(t => t.test), runs: list.map(t => t.run) });
}
(report as any)[kWorkerListsSymbol] = data;
}
return data;
}