Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(ui): llm conversation about error (#34750)

Simon Knottfe0b32777051
diff --git a/package-lock.json b/package-lock.json
index eb69ce2f36..e1edd82bf3 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -59,6 +59,7 @@
"formidable": "^2.1.1",
"immutable": "^4.3.7",
"license-checker": "^25.0.1",
+ "markdown-to-jsx": "^7.7.3",
"mime": "^3.0.0",
"node-stream-zip": "^1.15.0",
"react": "^18.1.0",
@@ -5897,7 +5898,6 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
"integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
- "dev": true,
"dependencies": {
"js-tokens": "^3.0.0 || ^4.0.0"
},
@@ -5953,6 +5953,18 @@
"semver": "bin/semver"
}
},
+ "node_modules/markdown-to-jsx": {
+ "version": "7.7.3",
+ "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-7.7.3.tgz",
+ "integrity": "sha512-o35IhJDFP6Fv60zPy+hbvZSQMmgvSGdK5j8NRZ7FeZMY+Bgqw+dSg7SC1ZEzC26++CiOUCqkbq96/c3j/FfTEQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 10"
+ },
+ "peerDependencies": {
+ "react": ">= 0.14.0"
+ }
+ },
"node_modules/matcher": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",
@@ -6784,7 +6796,6 @@
"version": "18.2.0",
"resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz",
"integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",
- "dev": true,
"dependencies": {
"loose-envify": "^1.1.0"
},
@@ -8975,6 +8986,7 @@
"packages/trace-viewer": {
"version": "0.0.0",
"dependencies": {
+ "markdown-to-jsx": "^7.7.3",
"yaml": "^2.6.0"
}
},
diff --git a/package.json b/package.json
index 8b54d54e6c..f27ad12a76 100644
--- a/package.json
+++ b/package.json
@@ -98,6 +98,7 @@
"formidable": "^2.1.1",
"immutable": "^4.3.7",
"license-checker": "^25.0.1",
+ "markdown-to-jsx": "^7.7.3",
"mime": "^3.0.0",
"node-stream-zip": "^1.15.0",
"react": "^18.1.0",
diff --git a/packages/playwright-core/src/server/trace/viewer/traceViewer.ts b/packages/playwright-core/src/server/trace/viewer/traceViewer.ts
index e0f05bac41..3f322e7125 100644
--- a/packages/playwright-core/src/server/trace/viewer/traceViewer.ts
+++ b/packages/playwright-core/src/server/trace/viewer/traceViewer.ts
@@ -142,6 +142,16 @@ export async function installRootRedirect(server: HttpServer, traceUrls: string[
server.routePath('/', (_, response) => {
response.statusCode = 302;
response.setHeader('Location', urlPath);
+
+ if (process.env.OPENAI_API_KEY)
+ response.appendHeader('Set-Cookie', `openai_api_key=${process.env.OPENAI_API_KEY}`);
+ if (process.env.OPENAI_BASE_URL)
+ response.appendHeader('Set-Cookie', `openai_base_url=${process.env.OPENAI_BASE_URL}`);
+ if (process.env.ANTHROPIC_API_KEY)
+ response.appendHeader('Set-Cookie', `anthropic_api_key=${process.env.ANTHROPIC_API_KEY}`);
+ if (process.env.ANTHROPIC_BASE_URL)
+ response.appendHeader('Set-Cookie', `anthropic_base_url=${process.env.ANTHROPIC_BASE_URL}`);
+
response.end();
return true;
});
diff --git a/packages/trace-viewer/package.json b/packages/trace-viewer/package.json
index 57a3dfd409..fde4eb6766 100644
--- a/packages/trace-viewer/package.json
+++ b/packages/trace-viewer/package.json
@@ -4,6 +4,7 @@
"version": "0.0.0",
"type": "module",
"dependencies": {
- "yaml": "^2.6.0"
+ "yaml": "^2.6.0",
+ "markdown-to-jsx": "^7.7.3"
}
}
diff --git a/packages/trace-viewer/src/sw/main.ts b/packages/trace-viewer/src/sw/main.ts
index cd23082d91..cdce3261fe 100644
--- a/packages/trace-viewer/src/sw/main.ts
+++ b/packages/trace-viewer/src/sw/main.ts
@@ -78,6 +78,12 @@ async function doFetch(event: FetchEvent): Promise<Response> {
if (event.request.url.startsWith('chrome-extension://'))
return fetch(event.request);
+ if (event.request.headers.get('x-pw-serviceworker') === 'forward') {
+ const request = new Request(event.request);
+ request.headers.delete('x-pw-serviceworker');
+ return fetch(request);
+ }
+
const request = event.request;
const client = await self.clients.get(event.clientId);
diff --git a/packages/trace-viewer/src/ui/aiConversation.css b/packages/trace-viewer/src/ui/aiConversation.css
new file mode 100644
index 0000000000..214496017a
--- /dev/null
+++ b/packages/trace-viewer/src/ui/aiConversation.css
@@ -0,0 +1,128 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.chat-container {
+ flex: 1;
+ display: flex;
+ flex-direction: column;
+ justify-content: space-between;
+ color: #e0e0e0;
+}
+
+.chat-disclaimer {
+ text-align: center;
+ color: var(--vscode-editorBracketMatch-border);
+ margin: 0px;
+}
+
+.chat-container hr {
+ width: 100%;
+ border: none;
+ border-top: 1px solid var(--vscode-titleBar-inactiveBackground);
+}
+
+.messages-container {
+ flex: 1;
+ overflow-y: auto;
+ padding: 16px;
+
+ display: flex;
+ flex-direction: column;
+ gap: 16px;
+}
+
+.message {
+ gap: 12px;
+ max-width: 85%;
+}
+
+.user-message {
+ flex-direction: row-reverse;
+ margin-left: auto;
+ width: fit-content
+}
+
+.message-icon {
+ width: 32px;
+ height: 32px;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ background-color: var(--vscode-titleBar-inactiveBackground);
+ flex-shrink: 0;
+}
+
+.message-content {
+ background-color: var(--vscode-titleBar-inactiveBackground);
+ color: var(--vscode-titleBar-activeForeground);
+ padding: 2px 8px;
+}
+
+.message-content pre {
+ text-wrap: auto;
+ overflow-wrap: anywhere;
+}
+
+.user-message .message-content {
+ background-color: var(--vscode-titleBar-activeBackground);
+}
+
+/* Input form styles */
+.input-form {
+ position: sticky;
+ bottom: 0;
+ display: flex;
+ height: 64px;
+ gap: 8px;
+ padding: 10px;
+ background-color: var(--vscode-sideBar-background);
+ border-top: 1px solid var(--vscode-sideBarSectionHeader-border);
+}
+
+.message-input {
+ flex: 1;
+ padding: 8px 12px;
+ border: 1px solid var(--vscode-settings-textInputBorder);
+ background-color: var(--vscode-settings-textInputBackground);
+ font-size: 14px;
+ outline: none;
+ transition: border-color 0.2s;
+}
+
+.message-input:focus {
+ border-color: #0078d4;
+}
+
+.send-button {
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ padding: 8px;
+ background-color: var(--vscode-button-background);
+ border: none;
+ color: white;
+ cursor: pointer;
+ transition: background-color 0.2s;
+}
+
+.send-button:hover {
+ background-color: var(--vscode-button-hoverBackground);
+}
+
+.send-button:disabled {
+ background-color: var(--vscode-disabledForeground);
+ cursor: not-allowed;
+}
diff --git a/packages/trace-viewer/src/ui/aiConversation.tsx b/packages/trace-viewer/src/ui/aiConversation.tsx
new file mode 100644
index 0000000000..548ed626f0
--- /dev/null
+++ b/packages/trace-viewer/src/ui/aiConversation.tsx
@@ -0,0 +1,84 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { useCallback, useState } from 'react';
+import Markdown from 'markdown-to-jsx';
+import './aiConversation.css';
+import { clsx } from '@web/uiUtils';
+import { useLLMConversation } from './llm';
+
+export function AIConversation({ conversationId }: { conversationId: string }) {
+ const [history, conversation] = useLLMConversation(conversationId);
+ const [input, setInput] = useState('');
+
+ const onSubmit = useCallback(() => {
+ setInput(content => {
+ conversation.send(content);
+ return '';
+ });
+ }, [conversation]);
+
+ return (
+ <div className='chat-container'>
+ <p className='chat-disclaimer'>Chat based on {conversation.chat.api.name}. Check for mistakes.</p>
+ <hr/>
+ <div className='messages-container'>
+ {history.filter(({ role }) => role !== 'developer').map((message, index) => (
+ <div
+ key={'' + index}
+ className={clsx('message', message.role === 'user' && 'user-message')}
+ >
+ {message.role === 'assistant' && (
+ <div className='message-icon'>
+ <img src='playwright-logo.svg' />
+ </div>
+ )}
+ <div className='message-content'>
+ <Markdown options={{ disableParsingRawHTML: true }}>{message.displayContent ?? message.content}</Markdown>
+ </div>
+ </div>
+ ))}
+ </div>
+
+ <div className='input-form'>
+ <textarea
+ name='content'
+ value={input}
+ onChange={e => setInput(e.target.value)}
+ onKeyDown={e => {
+ if (e.key === 'Enter' && !e.shiftKey) {
+ e.preventDefault();
+ onSubmit();
+ }
+ }}
+ placeholder='Ask a question...'
+ className='message-input'
+ />
+ {conversation.isSending() ? (
+ <button type='button' className='send-button' onClick={evt => {
+ evt.preventDefault();
+ conversation.abortSending();
+ }}>
+ Cancel
+ </button>
+ ) : (
+ <button className='send-button' disabled={!input.trim()} onClick={onSubmit}>
+ Send
+ </button>
+ )}
+ </div>
+ </div>
+ );
+}
diff --git a/packages/trace-viewer/src/ui/errorsTab.tsx b/packages/trace-viewer/src/ui/errorsTab.tsx
index 3d8651f74d..6142a0c83e 100644
--- a/packages/trace-viewer/src/ui/errorsTab.tsx
+++ b/packages/trace-viewer/src/ui/errorsTab.tsx
@@ -25,6 +25,10 @@ import { CopyToClipboardTextButton } from './copyToClipboard';
import { attachmentURL } from './attachmentsTab';
import { fixTestPrompt } from '@web/components/prompts';
import type { GitCommitInfo } from '@testIsomorphic/types';
+import { AIConversation } from './aiConversation';
+import { ToolbarButton } from '@web/components/toolbarButton';
+import { useIsLLMAvailable, useLLMChat } from './llm';
+import { useAsyncMemo } from '@web/uiUtils';
const GitCommitInfoContext = React.createContext<GitCommitInfo | undefined>(undefined);
@@ -36,33 +40,31 @@ export function useGitCommitInfo() {
return React.useContext(GitCommitInfoContext);
}
-const PromptButton: React.FC<{
- error: string;
- actions: modelUtil.ActionTraceEventInContext[];
-}> = ({ error, actions }) => {
- const [pageSnapshot, setPageSnapshot] = React.useState<string>();
-
- React.useEffect(() => {
+function usePageSnapshot(actions: modelUtil.ActionTraceEventInContext[]) {
+ return useAsyncMemo<string | undefined>(async () => {
for (const action of actions) {
for (const attachment of action.attachments ?? []) {
if (attachment.name === 'pageSnapshot') {
- fetch(attachmentURL({ ...attachment, traceUrl: action.context.traceUrl })).then(async response => {
- setPageSnapshot(await response.text());
- });
- return;
+ const response = await fetch(attachmentURL({ ...attachment, traceUrl: action.context.traceUrl }));
+ return await response.text();
}
}
}
- }, [actions]);
+ }, [actions], undefined);
+}
- const gitCommitInfo = useGitCommitInfo();
+const CopyPromptButton: React.FC<{
+ error: string;
+ pageSnapshot?: string;
+ diff?: string;
+}> = ({ error, pageSnapshot, diff }) => {
const prompt = React.useMemo(
() => fixTestPrompt(
error,
- gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'],
+ diff,
pageSnapshot
),
- [error, gitCommitInfo, pageSnapshot]
+ [error, diff, pageSnapshot]
);
return (
@@ -95,43 +97,101 @@ export function useErrorsTabModel(model: modelUtil.MultiTraceModel | undefined):
}, [model]);
}
+function Error({ message, error, errorId, sdkLanguage, pageSnapshot, revealInSource }: { message: string, error: ErrorDescription, errorId: string, sdkLanguage: Language, pageSnapshot?: string, revealInSource: (error: ErrorDescription) => void }) {
+ const [showLLM, setShowLLM] = React.useState(false);
+ const llmAvailable = useIsLLMAvailable();
+ const gitCommitInfo = useGitCommitInfo();
+ const diff = gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'];
+
+ let location: string | undefined;
+ let longLocation: string | undefined;
+ const stackFrame = error.stack?.[0];
+ if (stackFrame) {
+ const file = stackFrame.file.replace(/.*[/\\](.*)/, '$1');
+ location = file + ':' + stackFrame.line;
+ longLocation = stackFrame.file + ':' + stackFrame.line;
+ }
+
+ return <div style={{ display: 'flex', flexDirection: 'column', overflowX: 'clip' }}>
+ <div className='hbox' style={{
+ alignItems: 'center',
+ padding: '5px 10px',
+ minHeight: 36,
+ fontWeight: 'bold',
+ color: 'var(--vscode-errorForeground)',
+ flex: 0,
+ }}>
+ {error.action && renderAction(error.action, { sdkLanguage })}
+ {location && <div className='action-location'>
+ @ <span title={longLocation} onClick={() => revealInSource(error)}>{location}</span>
+ </div>}
+ <span style={{ position: 'absolute', right: '5px' }}>
+ {llmAvailable
+ ? <FixWithAIButton conversationId={errorId} onChange={setShowLLM} value={showLLM} error={message} diff={diff} pageSnapshot={pageSnapshot} />
+ : <CopyPromptButton error={message} pageSnapshot={pageSnapshot} diff={diff} />}
+ </span>
+ </div>
+
+ <ErrorMessage error={message} />
+
+ {showLLM && <AIConversation conversationId={errorId} />}
+ </div>;
+}
+
+function FixWithAIButton({ conversationId, value, onChange, error, diff, pageSnapshot }: { conversationId: string, value: boolean, onChange: React.Dispatch<React.SetStateAction<boolean>>, error: string, diff?: string, pageSnapshot?: string }) {
+ const chat = useLLMChat();
+
+ return <ToolbarButton
+ onClick={() => {
+ if (!chat.getConversation(conversationId)) {
+ const conversation = chat.startConversation(conversationId, [
+ `My Playwright test failed. What's going wrong?`,
+ `Please give me a suggestion how to fix it, and then explain what went wrong. Be very concise and apply Playwright best practices.`,
+ `Don't include many headings in your output. Make sure what you're saying is correct, and take into account whether there might be a bug in the app.`
+ ].join('\n'));
+
+ let content = `Here's the error: ${error}`;
+ let displayContent = `Help me with the error above.`;
+
+ if (diff)
+ content += `\n\nCode diff:\n${diff}`;
+ if (pageSnapshot)
+ content += `\n\nPage snapshot:\n${pageSnapshot}`;
+
+ if (diff)
+ displayContent += ` Take the code diff${pageSnapshot ? ' and page snapshot' : ''} into account.`;
+ else if (pageSnapshot)
+ displayContent += ` Take the page snapshot into account.`;
+
+ conversation.send(content, displayContent);
+ }
+
+ onChange(v => !v);
+ }}
+ style={{ width: '96px', justifyContent: 'center' }}
+ title='Fix with AI'
+ className='copy-to-clipboard-text-button'
+ >
+ {value ? 'Hide AI' : 'Fix with AI'}
+ </ToolbarButton>;
+}
+
export const ErrorsTab: React.FunctionComponent<{
errorsModel: ErrorsTabModel,
actions: modelUtil.ActionTraceEventInContext[],
+ wallTime: number,
sdkLanguage: Language,
revealInSource: (error: ErrorDescription) => void,
-}> = ({ errorsModel, sdkLanguage, revealInSource, actions }) => {
+}> = ({ errorsModel, sdkLanguage, revealInSource, actions, wallTime }) => {
+ const pageSnapshot = usePageSnapshot(actions);
+
if (!errorsModel.errors.size)
return <PlaceholderPanel text='No errors' />;
return <div className='fill' style={{ overflow: 'auto' }}>
{[...errorsModel.errors.entries()].map(([message, error]) => {
- let location: string | undefined;
- let longLocation: string | undefined;
- const stackFrame = error.stack?.[0];
- if (stackFrame) {
- const file = stackFrame.file.replace(/.*[/\\](.*)/, '$1');
- location = file + ':' + stackFrame.line;
- longLocation = stackFrame.file + ':' + stackFrame.line;
- }
- return <div key={message}>
- <div className='hbox' style={{
- alignItems: 'center',
- padding: '5px 10px',
- minHeight: 36,
- fontWeight: 'bold',
- color: 'var(--vscode-errorForeground)',
- }}>
- {error.action && renderAction(error.action, { sdkLanguage })}
- {location && <div className='action-location'>
- @ <span title={longLocation} onClick={() => revealInSource(error)}>{location}</span>
- </div>}
- <span style={{ position: 'absolute', right: '5px' }}>
- <PromptButton error={message} actions={actions} />
- </span>
- </div>
- <ErrorMessage error={message} />
- </div>;
+ const errorId = `error-${wallTime}-${message}`;
+ return <Error key={errorId} errorId={errorId} message={message} error={error} revealInSource={revealInSource} sdkLanguage={sdkLanguage} pageSnapshot={pageSnapshot} />;
})}
</div>;
};
diff --git a/packages/trace-viewer/src/ui/llm.tsx b/packages/trace-viewer/src/ui/llm.tsx
new file mode 100644
index 0000000000..c4093c1d51
--- /dev/null
+++ b/packages/trace-viewer/src/ui/llm.tsx
@@ -0,0 +1,267 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as React from 'react';
+import { EventEmitter } from '@testIsomorphic/events';
+import { useCookies } from '@web/uiUtils';
+
+export type LLMMessage = {
+ role: 'user' | 'assistant' | 'developer';
+ content: string;
+ displayContent?: string;
+};
+
+interface LLM {
+ readonly name: string;
+ chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string>;
+}
+
+// https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream
+async function *parseSSE(body: NonNullable<Response['body']>): AsyncGenerator<{ type: string, data: string, id: string }> {
+ const reader = body.pipeThrough(new TextDecoderStream()).getReader();
+ let buffer = '';
+
+ let lastEventId = '';
+ let type: string = '';
+ let data = '';
+
+ while (true) {
+ const { value, done } = await reader.read();
+ if (done)
+ break;
+ buffer += value;
+ const lines = buffer.split('\n');
+ buffer = lines.pop()!; // last line is either empty or incomplete
+
+ for (const line of lines) {
+ if (line.length === 0) {
+ if (data === '') {
+ data = '';
+ type = '';
+ continue;
+ }
+
+ if (data[data.length - 1] === '\n')
+ data = data.substring(0, data.length - 1);
+
+ const event = { type: type || 'message', data, id: lastEventId };
+ type = '';
+ data = '';
+
+ yield event;
+ }
+ if (line[0] === ':')
+ continue;
+
+ let name = '';
+ let value = '';
+ const colon = line.indexOf(':');
+ if (colon === -1) {
+ name = line;
+ } else {
+ name = line.substring(0, colon);
+ value = line[colon + 1] === ' ' ? line.substring(colon + 2) : line.substring(colon + 1);
+ }
+
+ switch (name) {
+ case 'event':
+ type = value;
+ break;
+ case 'data':
+ data += value + '\n';
+ break;
+ case 'id':
+ lastEventId = value;
+ break;
+ case 'retry':
+ default:
+ // not implemented
+ break;
+ }
+ }
+ }
+}
+
+class OpenAI implements LLM {
+
+ name = 'OpenAI';
+
+ constructor(private apiKey: string, private baseURL = 'https://api.openai.com') {}
+
+ async *chatCompletion(messages: LLMMessage[], signal: AbortSignal) {
+ const url = new URL('./v1/chat/completions', this.baseURL);
+ const response = await fetch(url, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'Authorization': `Bearer ${this.apiKey}`,
+ 'x-pw-serviceworker': 'forward',
+ },
+ body: JSON.stringify({
+ model: 'gpt-4o', // TODO: make configurable
+ messages: messages.map(({ role, content }) => ({ role, content })),
+ stream: true,
+ }),
+ signal,
+ });
+
+ if (response.status !== 200 || !response.body)
+ throw new Error('Failed to chat with OpenAI, unexpected status: ' + response.status + await response.text());
+
+ for await (const sseEvent of parseSSE(response.body)) {
+ const event = JSON.parse(sseEvent.data);
+ if (event.object === 'chat.completion.chunk') {
+ if (event.choices[0].finish_reason)
+ break;
+ yield event.choices[0].delta.content;
+ }
+ }
+ }
+}
+
+class Anthropic implements LLM {
+ name = 'Anthropic';
+ constructor(private apiKey: string, private baseURL = 'https://api.anthropic.com') {}
+ async *chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string> {
+ const response = await fetch(new URL('./v1/messages', this.baseURL), {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ 'x-api-key': this.apiKey,
+ 'anthropic-version': '2023-06-01',
+ 'x-pw-serviceworker': 'forward',
+ },
+ body: JSON.stringify({
+ model: 'claude-3-5-sonnet-20241022', // TODO: make configurable
+ messages: messages.filter(({ role }) => role !== 'developer').map(({ role, content }) => ({ role, content })),
+ system: messages.find(({ role }) => role === 'developer')?.content,
+ max_tokens: 1024,
+ stream: true,
+ }),
+ signal,
+ });
+
+ if (response.status !== 200 || !response.body)
+ throw new Error('Failed to chat with Anthropic, unexpected status: ' + response.status + await response.text());
+
+ for await (const sseEvent of parseSSE(response.body)) {
+ const event = JSON.parse(sseEvent.data);
+ if (event.type === 'content_block_delta')
+ yield event.delta.text;
+ }
+ }
+}
+
+class LLMChat {
+ conversations = new Map<string, Conversation>();
+
+ constructor(readonly api: LLM) {}
+
+ getConversation(id: string) {
+ return this.conversations.get(id);
+ }
+
+ startConversation(id: string, systemPrompt: string) {
+ const conversation = new Conversation(this, systemPrompt);
+ this.conversations.set(id, conversation); // TODO: cleanup
+ return conversation;
+ }
+}
+
+export class Conversation {
+ history: LLMMessage[];
+ onChange = new EventEmitter<void>();
+ private _abortControllers = new Set<AbortController>();
+
+ constructor(public chat: LLMChat, systemPrompt: string) {
+ this.history = [{ role: 'developer', content: systemPrompt }];
+ }
+
+ async send(content: string, displayContent?: string) {
+ const response: LLMMessage = { role: 'assistant', content: '' };
+ this.history.push({ role: 'user', content, displayContent }, response);
+ const abortController = new AbortController();
+ this._abortControllers.add(abortController);
+ this.onChange.fire();
+ try {
+ for await (const chunk of this.chat.api.chatCompletion(this.history, abortController.signal)) {
+ response.content += chunk;
+ this.onChange.fire();
+ }
+ } finally {
+ this._abortControllers.delete(abortController);
+ this.onChange.fire();
+ }
+ }
+
+ isSending(): boolean {
+ return this._abortControllers.size > 0;
+ }
+
+ abortSending() {
+ for (const controller of this._abortControllers)
+ controller.abort();
+ this._abortControllers.clear();
+ this.onChange.fire();
+ }
+
+ isEmpty() {
+ return this.history.length < 2;
+ }
+}
+
+
+const llmContext = React.createContext<LLMChat | undefined>(undefined);
+
+export function LLMProvider({ children }: React.PropsWithChildren<{}>) {
+ const cookiePairs = useCookies();
+ const chat = React.useMemo(() => {
+ const cookies = Object.fromEntries(cookiePairs);
+ if (cookies.openai_api_key)
+ return new LLMChat(new OpenAI(cookies.openai_api_key, cookies.openai_base_url));
+ if (cookies.anthropic_api_key)
+ return new LLMChat(new Anthropic(cookies.anthropic_api_key, cookies.anthropic_base_url));
+ }, [cookiePairs]);
+ return <llmContext.Provider value={chat}>{children}</llmContext.Provider>;
+}
+
+export function useLLMChat() {
+ const chat = React.useContext(llmContext);
+ if (!chat)
+ throw new Error('No LLM chat available, make sure theres a LLMProvider above');
+ return chat;
+}
+
+export function useIsLLMAvailable() {
+ return !!React.useContext(llmContext);
+}
+
+export function useLLMConversation(id: string) {
+ const conversation = useLLMChat().getConversation(id);
+ if (!conversation)
+ throw new Error('No conversation found for id: ' + id);
+ const [history, setHistory] = React.useState(conversation.history);
+ React.useEffect(() => {
+ function update() {
+ setHistory([...conversation!.history]);
+ }
+ update();
+ const subscription = conversation.onChange.event(update);
+ return subscription.dispose;
+ }, [conversation]);
+
+ return [history, conversation] as const;
+}
diff --git a/packages/trace-viewer/src/ui/uiModeView.tsx b/packages/trace-viewer/src/ui/uiModeView.tsx
index 8e27ed0137..441163c5d9 100644
--- a/packages/trace-viewer/src/ui/uiModeView.tsx
+++ b/packages/trace-viewer/src/ui/uiModeView.tsx
@@ -38,6 +38,7 @@ import { TraceView } from './uiModeTraceView';
import { SettingsView } from './settingsView';
import { DefaultSettingsView } from './defaultSettingsView';
import { GitCommitInfoProvider } from './errorsTab';
+import { LLMProvider } from './llm';
let xtermSize = { cols: 80, rows: 24 };
const xtermDataSource: XtermDataSource = {
@@ -398,7 +399,7 @@ export const UIModeView: React.FC<{}> = ({
});
}, [closeInstallDialog, testServerConnection]);
- return <div className='vbox ui-mode'>
+ return <LLMProvider><div className='vbox ui-mode'>
{!hasBrowsers && <dialog ref={dialogRef}>
<div className='title'><span className='codicon codicon-lightbulb'></span>Install browsers</div>
<div className='body'>
@@ -527,5 +528,5 @@ export const UIModeView: React.FC<{}> = ({
</div>
}
/>
- </div>;
+ </div></LLMProvider>;
};
diff --git a/packages/trace-viewer/src/ui/workbench.tsx b/packages/trace-viewer/src/ui/workbench.tsx
index 25d01098ed..34f27d65cb 100644
--- a/packages/trace-viewer/src/ui/workbench.tsx
+++ b/packages/trace-viewer/src/ui/workbench.tsx
@@ -199,7 +199,7 @@ export const Workbench: React.FunctionComponent<{
else
setRevealedError(error);
selectPropertiesTab('source');
- }} actions={model?.actions ?? []} />
+ }} actions={model?.actions ?? []} wallTime={model?.wallTime ?? 0} />
};
// Fallback location w/o action stands for file / test.
diff --git a/packages/trace-viewer/src/ui/workbenchLoader.tsx b/packages/trace-viewer/src/ui/workbenchLoader.tsx
index 9f71eb1142..9a90523f9a 100644
--- a/packages/trace-viewer/src/ui/workbenchLoader.tsx
+++ b/packages/trace-viewer/src/ui/workbenchLoader.tsx
@@ -21,6 +21,7 @@ import './workbenchLoader.css';
import { Workbench } from './workbench';
import { TestServerConnection, WebSocketTestServerTransport } from '@testIsomorphic/testServerConnection';
import { SettingsToolbarButton } from './settingsToolbarButton';
+import { LLMProvider } from './llm';
export const WorkbenchLoader: React.FunctionComponent<{
}> = () => {
@@ -165,7 +166,9 @@ export const WorkbenchLoader: React.FunctionComponent<{
<div className='progress'>
<div className='inner-progress' style={{ width: progress.total ? (100 * progress.done / progress.total) + '%' : 0 }}></div>
</div>
- <Workbench model={model} inert={showFileUploadDropArea} />
+ <LLMProvider>
+ <Workbench model={model} inert={showFileUploadDropArea} />
+ </LLMProvider>
{fileForLocalModeError && <div className='drop-target'>
<div>Trace Viewer uses Service Workers to show traces. To view trace:</div>
<div style={{ paddingTop: 20 }}>
diff --git a/packages/web/src/uiUtils.ts b/packages/web/src/uiUtils.ts
index a0b7a59c36..cded19af20 100644
--- a/packages/web/src/uiUtils.ts
+++ b/packages/web/src/uiUtils.ts
@@ -248,3 +248,13 @@ export function useFlash(): [boolean, EffectCallback] {
}, [setFlash]);
return [flash, trigger];
}
+
+export function useCookies() {
+ const cookies = React.useMemo(() => {
+ return document.cookie.split('; ').filter(v => v.includes('=')).map(kv => {
+ const separator = kv.indexOf('=');
+ return [kv.substring(0, separator), kv.substring(separator + 1)];
+ });
+ }, []);
+ return cookies;
+}
diff --git a/tests/playwright-test/ui-mode-llm.spec.ts b/tests/playwright-test/ui-mode-llm.spec.ts
new file mode 100644
index 0000000000..8c7b0447ec
--- /dev/null
+++ b/tests/playwright-test/ui-mode-llm.spec.ts
@@ -0,0 +1,99 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, retries } from './ui-mode-fixtures';
+
+test.describe.configure({ mode: 'parallel', retries });
+
+test('openai', async ({ runUITest, server }) => {
+ server.setRoute('/v1/chat/completions', async (req, res) => {
+ res.setHeader('Access-Control-Allow-Origin', '*');
+ res.setHeader('Access-Control-Allow-Headers', '*');
+ if (req.method === 'OPTIONS')
+ return res.end();
+
+ expect(req.headers.authorization).toBe('Bearer fake-key');
+ expect((await req.postBody).toString()).toContain(`- button \\"Submit\\"`);
+ const event = {
+ object: 'chat.completion.chunk',
+ choices: [{ delta: { content: 'This is a mock response' } }]
+ };
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.write(`data: ${JSON.stringify(event)}\n\n`);
+ });
+
+ const { page } = await runUITest({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('trace test', async ({ page }) => {
+ await page.setContent('<button>Submit</button>');
+ expect(1).toBe(2);
+ });
+ `,
+ }, {
+ OPENAI_API_KEY: 'fake-key',
+ OPENAI_BASE_URL: server.PREFIX,
+ });
+
+ await page.getByTitle('Run all').click();
+ await page.getByText('Errors', { exact: true }).click();
+ await page.getByRole('button', { name: 'Fix with AI' }).click();
+ await expect(page.getByRole('tabpanel', { name: 'Errors' })).toMatchAriaSnapshot(`
+ - tabpanel "Errors":
+ - text: Help me with the error above. Take the page snapshot into account.
+ - text: This is a mock response
+ `);
+});
+
+test('anthropic', async ({ runUITest, server }) => {
+ server.setRoute('/v1/messages', async (req, res) => {
+ res.setHeader('Access-Control-Allow-Origin', '*');
+ res.setHeader('Access-Control-Allow-Headers', '*');
+ if (req.method === 'OPTIONS')
+ return res.end();
+
+ expect(req.headers['x-api-key']).toBe('fake-key');
+ expect((await req.postBody).toString()).toContain(`- button \\"Submit\\"`);
+ const event = {
+ type: 'content_block_delta',
+ delta: { text: 'This is a mock response' },
+ };
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.write(`data: ${JSON.stringify(event)}\n\n`);
+ });
+
+ const { page } = await runUITest({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('trace test', async ({ page }) => {
+ await page.setContent('<button>Submit</button>');
+ expect(1).toBe(2);
+ });
+ `,
+ }, {
+ ANTHROPIC_API_KEY: 'fake-key',
+ ANTHROPIC_BASE_URL: server.PREFIX,
+ });
+
+ await page.getByTitle('Run all').click();
+ await page.getByText('Errors', { exact: true }).click();
+ await page.getByRole('button', { name: 'Fix with AI' }).click();
+ await expect(page.getByRole('tabpanel', { name: 'Errors' })).toMatchAriaSnapshot(`
+ - tabpanel "Errors":
+ - text: Help me with the error above. Take the page snapshot into account.
+ - text: This is a mock response
+ `);
+});