Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(ui): llm conversation about error (#34750)
Simon Knottfe0b32777051
diff --git a/package-lock.json b/package-lock.jsonindex eb69ce2f36..e1edd82bf3 100644--- a/package-lock.json+++ b/package-lock.json@@ -59,6 +59,7 @@"formidable": "^2.1.1","immutable": "^4.3.7","license-checker": "^25.0.1",+ "markdown-to-jsx": "^7.7.3","mime": "^3.0.0","node-stream-zip": "^1.15.0","react": "^18.1.0",@@ -5897,7 +5898,6 @@"version": "1.4.0","resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz","integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",- "dev": true,"dependencies": {"js-tokens": "^3.0.0 || ^4.0.0"},@@ -5953,6 +5953,18 @@"semver": "bin/semver"}},+ "node_modules/markdown-to-jsx": {+ "version": "7.7.3",+ "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-7.7.3.tgz",+ "integrity": "sha512-o35IhJDFP6Fv60zPy+hbvZSQMmgvSGdK5j8NRZ7FeZMY+Bgqw+dSg7SC1ZEzC26++CiOUCqkbq96/c3j/FfTEQ==",+ "license": "MIT",+ "engines": {+ "node": ">= 10"+ },+ "peerDependencies": {+ "react": ">= 0.14.0"+ }+ },"node_modules/matcher": {"version": "3.0.0","resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz",@@ -6784,7 +6796,6 @@"version": "18.2.0","resolved": "https://registry.npmjs.org/react/-/react-18.2.0.tgz","integrity": "sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==",- "dev": true,"dependencies": {"loose-envify": "^1.1.0"},@@ -8975,6 +8986,7 @@"packages/trace-viewer": {"version": "0.0.0","dependencies": {+ "markdown-to-jsx": "^7.7.3","yaml": "^2.6.0"}},diff --git a/package.json b/package.jsonindex 8b54d54e6c..f27ad12a76 100644--- a/package.json+++ b/package.json@@ -98,6 +98,7 @@"formidable": "^2.1.1","immutable": "^4.3.7","license-checker": "^25.0.1",+ "markdown-to-jsx": "^7.7.3","mime": "^3.0.0","node-stream-zip": "^1.15.0","react": "^18.1.0",diff --git a/packages/playwright-core/src/server/trace/viewer/traceViewer.ts b/packages/playwright-core/src/server/trace/viewer/traceViewer.tsindex e0f05bac41..3f322e7125 100644--- a/packages/playwright-core/src/server/trace/viewer/traceViewer.ts+++ b/packages/playwright-core/src/server/trace/viewer/traceViewer.ts@@ -142,6 +142,16 @@ export async function installRootRedirect(server: HttpServer, traceUrls: string[server.routePath('/', (_, response) => {response.statusCode = 302;response.setHeader('Location', urlPath);++ if (process.env.OPENAI_API_KEY)+ response.appendHeader('Set-Cookie', `openai_api_key=${process.env.OPENAI_API_KEY}`);+ if (process.env.OPENAI_BASE_URL)+ response.appendHeader('Set-Cookie', `openai_base_url=${process.env.OPENAI_BASE_URL}`);+ if (process.env.ANTHROPIC_API_KEY)+ response.appendHeader('Set-Cookie', `anthropic_api_key=${process.env.ANTHROPIC_API_KEY}`);+ if (process.env.ANTHROPIC_BASE_URL)+ response.appendHeader('Set-Cookie', `anthropic_base_url=${process.env.ANTHROPIC_BASE_URL}`);+response.end();return true;});diff --git a/packages/trace-viewer/package.json b/packages/trace-viewer/package.jsonindex 57a3dfd409..fde4eb6766 100644--- a/packages/trace-viewer/package.json+++ b/packages/trace-viewer/package.json@@ -4,6 +4,7 @@"version": "0.0.0","type": "module","dependencies": {- "yaml": "^2.6.0"+ "yaml": "^2.6.0",+ "markdown-to-jsx": "^7.7.3"}}diff --git a/packages/trace-viewer/src/sw/main.ts b/packages/trace-viewer/src/sw/main.tsindex cd23082d91..cdce3261fe 100644--- a/packages/trace-viewer/src/sw/main.ts+++ b/packages/trace-viewer/src/sw/main.ts@@ -78,6 +78,12 @@ async function doFetch(event: FetchEvent): Promise<Response> {if (event.request.url.startsWith('chrome-extension://'))return fetch(event.request);+ if (event.request.headers.get('x-pw-serviceworker') === 'forward') {+ const request = new Request(event.request);+ request.headers.delete('x-pw-serviceworker');+ return fetch(request);+ }+const request = event.request;const client = await self.clients.get(event.clientId);diff --git a/packages/trace-viewer/src/ui/aiConversation.css b/packages/trace-viewer/src/ui/aiConversation.cssnew file mode 100644index 0000000000..214496017a--- /dev/null+++ b/packages/trace-viewer/src/ui/aiConversation.css@@ -0,0 +1,128 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++.chat-container {+ flex: 1;+ display: flex;+ flex-direction: column;+ justify-content: space-between;+ color: #e0e0e0;+}++.chat-disclaimer {+ text-align: center;+ color: var(--vscode-editorBracketMatch-border);+ margin: 0px;+}++.chat-container hr {+ width: 100%;+ border: none;+ border-top: 1px solid var(--vscode-titleBar-inactiveBackground);+}++.messages-container {+ flex: 1;+ overflow-y: auto;+ padding: 16px;++ display: flex;+ flex-direction: column;+ gap: 16px;+}++.message {+ gap: 12px;+ max-width: 85%;+}++.user-message {+ flex-direction: row-reverse;+ margin-left: auto;+ width: fit-content+}++.message-icon {+ width: 32px;+ height: 32px;+ display: flex;+ align-items: center;+ justify-content: center;+ background-color: var(--vscode-titleBar-inactiveBackground);+ flex-shrink: 0;+}++.message-content {+ background-color: var(--vscode-titleBar-inactiveBackground);+ color: var(--vscode-titleBar-activeForeground);+ padding: 2px 8px;+}++.message-content pre {+ text-wrap: auto;+ overflow-wrap: anywhere;+}++.user-message .message-content {+ background-color: var(--vscode-titleBar-activeBackground);+}++/* Input form styles */+.input-form {+ position: sticky;+ bottom: 0;+ display: flex;+ height: 64px;+ gap: 8px;+ padding: 10px;+ background-color: var(--vscode-sideBar-background);+ border-top: 1px solid var(--vscode-sideBarSectionHeader-border);+}++.message-input {+ flex: 1;+ padding: 8px 12px;+ border: 1px solid var(--vscode-settings-textInputBorder);+ background-color: var(--vscode-settings-textInputBackground);+ font-size: 14px;+ outline: none;+ transition: border-color 0.2s;+}++.message-input:focus {+ border-color: #0078d4;+}++.send-button {+ display: flex;+ align-items: center;+ justify-content: center;+ padding: 8px;+ background-color: var(--vscode-button-background);+ border: none;+ color: white;+ cursor: pointer;+ transition: background-color 0.2s;+}++.send-button:hover {+ background-color: var(--vscode-button-hoverBackground);+}++.send-button:disabled {+ background-color: var(--vscode-disabledForeground);+ cursor: not-allowed;+}diff --git a/packages/trace-viewer/src/ui/aiConversation.tsx b/packages/trace-viewer/src/ui/aiConversation.tsxnew file mode 100644index 0000000000..548ed626f0--- /dev/null+++ b/packages/trace-viewer/src/ui/aiConversation.tsx@@ -0,0 +1,84 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+import { useCallback, useState } from 'react';+import Markdown from 'markdown-to-jsx';+import './aiConversation.css';+import { clsx } from '@web/uiUtils';+import { useLLMConversation } from './llm';++export function AIConversation({ conversationId }: { conversationId: string }) {+ const [history, conversation] = useLLMConversation(conversationId);+ const [input, setInput] = useState('');++ const onSubmit = useCallback(() => {+ setInput(content => {+ conversation.send(content);+ return '';+ });+ }, [conversation]);++ return (+ <div className='chat-container'>+ <p className='chat-disclaimer'>Chat based on {conversation.chat.api.name}. Check for mistakes.</p>+ <hr/>+ <div className='messages-container'>+ {history.filter(({ role }) => role !== 'developer').map((message, index) => (+ <div+ key={'' + index}+ className={clsx('message', message.role === 'user' && 'user-message')}+ >+ {message.role === 'assistant' && (+ <div className='message-icon'>+ <img src='playwright-logo.svg' />+ </div>+ )}+ <div className='message-content'>+ <Markdown options={{ disableParsingRawHTML: true }}>{message.displayContent ?? message.content}</Markdown>+ </div>+ </div>+ ))}+ </div>++ <div className='input-form'>+ <textarea+ name='content'+ value={input}+ onChange={e => setInput(e.target.value)}+ onKeyDown={e => {+ if (e.key === 'Enter' && !e.shiftKey) {+ e.preventDefault();+ onSubmit();+ }+ }}+ placeholder='Ask a question...'+ className='message-input'+ />+ {conversation.isSending() ? (+ <button type='button' className='send-button' onClick={evt => {+ evt.preventDefault();+ conversation.abortSending();+ }}>+ Cancel+ </button>+ ) : (+ <button className='send-button' disabled={!input.trim()} onClick={onSubmit}>+ Send+ </button>+ )}+ </div>+ </div>+ );+}diff --git a/packages/trace-viewer/src/ui/errorsTab.tsx b/packages/trace-viewer/src/ui/errorsTab.tsxindex 3d8651f74d..6142a0c83e 100644--- a/packages/trace-viewer/src/ui/errorsTab.tsx+++ b/packages/trace-viewer/src/ui/errorsTab.tsx@@ -25,6 +25,10 @@ import { CopyToClipboardTextButton } from './copyToClipboard';import { attachmentURL } from './attachmentsTab';import { fixTestPrompt } from '@web/components/prompts';import type { GitCommitInfo } from '@testIsomorphic/types';+import { AIConversation } from './aiConversation';+import { ToolbarButton } from '@web/components/toolbarButton';+import { useIsLLMAvailable, useLLMChat } from './llm';+import { useAsyncMemo } from '@web/uiUtils';const GitCommitInfoContext = React.createContext<GitCommitInfo | undefined>(undefined);@@ -36,33 +40,31 @@ export function useGitCommitInfo() {return React.useContext(GitCommitInfoContext);}-const PromptButton: React.FC<{- error: string;- actions: modelUtil.ActionTraceEventInContext[];-}> = ({ error, actions }) => {- const [pageSnapshot, setPageSnapshot] = React.useState<string>();-- React.useEffect(() => {+function usePageSnapshot(actions: modelUtil.ActionTraceEventInContext[]) {+ return useAsyncMemo<string | undefined>(async () => {for (const action of actions) {for (const attachment of action.attachments ?? []) {if (attachment.name === 'pageSnapshot') {- fetch(attachmentURL({ ...attachment, traceUrl: action.context.traceUrl })).then(async response => {- setPageSnapshot(await response.text());- });- return;+ const response = await fetch(attachmentURL({ ...attachment, traceUrl: action.context.traceUrl }));+ return await response.text();}}}- }, [actions]);+ }, [actions], undefined);+}- const gitCommitInfo = useGitCommitInfo();+const CopyPromptButton: React.FC<{+ error: string;+ pageSnapshot?: string;+ diff?: string;+}> = ({ error, pageSnapshot, diff }) => {const prompt = React.useMemo(() => fixTestPrompt(error,- gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'],+ diff,pageSnapshot),- [error, gitCommitInfo, pageSnapshot]+ [error, diff, pageSnapshot]);return (@@ -95,43 +97,101 @@ export function useErrorsTabModel(model: modelUtil.MultiTraceModel | undefined):}, [model]);}+function Error({ message, error, errorId, sdkLanguage, pageSnapshot, revealInSource }: { message: string, error: ErrorDescription, errorId: string, sdkLanguage: Language, pageSnapshot?: string, revealInSource: (error: ErrorDescription) => void }) {+ const [showLLM, setShowLLM] = React.useState(false);+ const llmAvailable = useIsLLMAvailable();+ const gitCommitInfo = useGitCommitInfo();+ const diff = gitCommitInfo?.['pull.diff'] ?? gitCommitInfo?.['revision.diff'];++ let location: string | undefined;+ let longLocation: string | undefined;+ const stackFrame = error.stack?.[0];+ if (stackFrame) {+ const file = stackFrame.file.replace(/.*[/\\](.*)/, '$1');+ location = file + ':' + stackFrame.line;+ longLocation = stackFrame.file + ':' + stackFrame.line;+ }++ return <div style={{ display: 'flex', flexDirection: 'column', overflowX: 'clip' }}>+ <div className='hbox' style={{+ alignItems: 'center',+ padding: '5px 10px',+ minHeight: 36,+ fontWeight: 'bold',+ color: 'var(--vscode-errorForeground)',+ flex: 0,+ }}>+ {error.action && renderAction(error.action, { sdkLanguage })}+ {location && <div className='action-location'>+ @ <span title={longLocation} onClick={() => revealInSource(error)}>{location}</span>+ </div>}+ <span style={{ position: 'absolute', right: '5px' }}>+ {llmAvailable+ ? <FixWithAIButton conversationId={errorId} onChange={setShowLLM} value={showLLM} error={message} diff={diff} pageSnapshot={pageSnapshot} />+ : <CopyPromptButton error={message} pageSnapshot={pageSnapshot} diff={diff} />}+ </span>+ </div>++ <ErrorMessage error={message} />++ {showLLM && <AIConversation conversationId={errorId} />}+ </div>;+}++function FixWithAIButton({ conversationId, value, onChange, error, diff, pageSnapshot }: { conversationId: string, value: boolean, onChange: React.Dispatch<React.SetStateAction<boolean>>, error: string, diff?: string, pageSnapshot?: string }) {+ const chat = useLLMChat();++ return <ToolbarButton+ onClick={() => {+ if (!chat.getConversation(conversationId)) {+ const conversation = chat.startConversation(conversationId, [+ `My Playwright test failed. What's going wrong?`,+ `Please give me a suggestion how to fix it, and then explain what went wrong. Be very concise and apply Playwright best practices.`,+ `Don't include many headings in your output. Make sure what you're saying is correct, and take into account whether there might be a bug in the app.`+ ].join('\n'));++ let content = `Here's the error: ${error}`;+ let displayContent = `Help me with the error above.`;++ if (diff)+ content += `\n\nCode diff:\n${diff}`;+ if (pageSnapshot)+ content += `\n\nPage snapshot:\n${pageSnapshot}`;++ if (diff)+ displayContent += ` Take the code diff${pageSnapshot ? ' and page snapshot' : ''} into account.`;+ else if (pageSnapshot)+ displayContent += ` Take the page snapshot into account.`;++ conversation.send(content, displayContent);+ }++ onChange(v => !v);+ }}+ style={{ width: '96px', justifyContent: 'center' }}+ title='Fix with AI'+ className='copy-to-clipboard-text-button'+ >+ {value ? 'Hide AI' : 'Fix with AI'}+ </ToolbarButton>;+}+export const ErrorsTab: React.FunctionComponent<{errorsModel: ErrorsTabModel,actions: modelUtil.ActionTraceEventInContext[],+ wallTime: number,sdkLanguage: Language,revealInSource: (error: ErrorDescription) => void,-}> = ({ errorsModel, sdkLanguage, revealInSource, actions }) => {+}> = ({ errorsModel, sdkLanguage, revealInSource, actions, wallTime }) => {+ const pageSnapshot = usePageSnapshot(actions);+if (!errorsModel.errors.size)return <PlaceholderPanel text='No errors' />;return <div className='fill' style={{ overflow: 'auto' }}>{[...errorsModel.errors.entries()].map(([message, error]) => {- let location: string | undefined;- let longLocation: string | undefined;- const stackFrame = error.stack?.[0];- if (stackFrame) {- const file = stackFrame.file.replace(/.*[/\\](.*)/, '$1');- location = file + ':' + stackFrame.line;- longLocation = stackFrame.file + ':' + stackFrame.line;- }- return <div key={message}>- <div className='hbox' style={{- alignItems: 'center',- padding: '5px 10px',- minHeight: 36,- fontWeight: 'bold',- color: 'var(--vscode-errorForeground)',- }}>- {error.action && renderAction(error.action, { sdkLanguage })}- {location && <div className='action-location'>- @ <span title={longLocation} onClick={() => revealInSource(error)}>{location}</span>- </div>}- <span style={{ position: 'absolute', right: '5px' }}>- <PromptButton error={message} actions={actions} />- </span>- </div>- <ErrorMessage error={message} />- </div>;+ const errorId = `error-${wallTime}-${message}`;+ return <Error key={errorId} errorId={errorId} message={message} error={error} revealInSource={revealInSource} sdkLanguage={sdkLanguage} pageSnapshot={pageSnapshot} />;})}</div>;};diff --git a/packages/trace-viewer/src/ui/llm.tsx b/packages/trace-viewer/src/ui/llm.tsxnew file mode 100644index 0000000000..c4093c1d51--- /dev/null+++ b/packages/trace-viewer/src/ui/llm.tsx@@ -0,0 +1,267 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import * as React from 'react';+import { EventEmitter } from '@testIsomorphic/events';+import { useCookies } from '@web/uiUtils';++export type LLMMessage = {+ role: 'user' | 'assistant' | 'developer';+ content: string;+ displayContent?: string;+};++interface LLM {+ readonly name: string;+ chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string>;+}++// https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream+async function *parseSSE(body: NonNullable<Response['body']>): AsyncGenerator<{ type: string, data: string, id: string }> {+ const reader = body.pipeThrough(new TextDecoderStream()).getReader();+ let buffer = '';++ let lastEventId = '';+ let type: string = '';+ let data = '';++ while (true) {+ const { value, done } = await reader.read();+ if (done)+ break;+ buffer += value;+ const lines = buffer.split('\n');+ buffer = lines.pop()!; // last line is either empty or incomplete++ for (const line of lines) {+ if (line.length === 0) {+ if (data === '') {+ data = '';+ type = '';+ continue;+ }++ if (data[data.length - 1] === '\n')+ data = data.substring(0, data.length - 1);++ const event = { type: type || 'message', data, id: lastEventId };+ type = '';+ data = '';++ yield event;+ }+ if (line[0] === ':')+ continue;++ let name = '';+ let value = '';+ const colon = line.indexOf(':');+ if (colon === -1) {+ name = line;+ } else {+ name = line.substring(0, colon);+ value = line[colon + 1] === ' ' ? line.substring(colon + 2) : line.substring(colon + 1);+ }++ switch (name) {+ case 'event':+ type = value;+ break;+ case 'data':+ data += value + '\n';+ break;+ case 'id':+ lastEventId = value;+ break;+ case 'retry':+ default:+ // not implemented+ break;+ }+ }+ }+}++class OpenAI implements LLM {++ name = 'OpenAI';++ constructor(private apiKey: string, private baseURL = 'https://api.openai.com') {}++ async *chatCompletion(messages: LLMMessage[], signal: AbortSignal) {+ const url = new URL('./v1/chat/completions', this.baseURL);+ const response = await fetch(url, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ 'Authorization': `Bearer ${this.apiKey}`,+ 'x-pw-serviceworker': 'forward',+ },+ body: JSON.stringify({+ model: 'gpt-4o', // TODO: make configurable+ messages: messages.map(({ role, content }) => ({ role, content })),+ stream: true,+ }),+ signal,+ });++ if (response.status !== 200 || !response.body)+ throw new Error('Failed to chat with OpenAI, unexpected status: ' + response.status + await response.text());++ for await (const sseEvent of parseSSE(response.body)) {+ const event = JSON.parse(sseEvent.data);+ if (event.object === 'chat.completion.chunk') {+ if (event.choices[0].finish_reason)+ break;+ yield event.choices[0].delta.content;+ }+ }+ }+}++class Anthropic implements LLM {+ name = 'Anthropic';+ constructor(private apiKey: string, private baseURL = 'https://api.anthropic.com') {}+ async *chatCompletion(messages: LLMMessage[], signal: AbortSignal): AsyncGenerator<string> {+ const response = await fetch(new URL('./v1/messages', this.baseURL), {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ 'x-api-key': this.apiKey,+ 'anthropic-version': '2023-06-01',+ 'x-pw-serviceworker': 'forward',+ },+ body: JSON.stringify({+ model: 'claude-3-5-sonnet-20241022', // TODO: make configurable+ messages: messages.filter(({ role }) => role !== 'developer').map(({ role, content }) => ({ role, content })),+ system: messages.find(({ role }) => role === 'developer')?.content,+ max_tokens: 1024,+ stream: true,+ }),+ signal,+ });++ if (response.status !== 200 || !response.body)+ throw new Error('Failed to chat with Anthropic, unexpected status: ' + response.status + await response.text());++ for await (const sseEvent of parseSSE(response.body)) {+ const event = JSON.parse(sseEvent.data);+ if (event.type === 'content_block_delta')+ yield event.delta.text;+ }+ }+}++class LLMChat {+ conversations = new Map<string, Conversation>();++ constructor(readonly api: LLM) {}++ getConversation(id: string) {+ return this.conversations.get(id);+ }++ startConversation(id: string, systemPrompt: string) {+ const conversation = new Conversation(this, systemPrompt);+ this.conversations.set(id, conversation); // TODO: cleanup+ return conversation;+ }+}++export class Conversation {+ history: LLMMessage[];+ onChange = new EventEmitter<void>();+ private _abortControllers = new Set<AbortController>();++ constructor(public chat: LLMChat, systemPrompt: string) {+ this.history = [{ role: 'developer', content: systemPrompt }];+ }++ async send(content: string, displayContent?: string) {+ const response: LLMMessage = { role: 'assistant', content: '' };+ this.history.push({ role: 'user', content, displayContent }, response);+ const abortController = new AbortController();+ this._abortControllers.add(abortController);+ this.onChange.fire();+ try {+ for await (const chunk of this.chat.api.chatCompletion(this.history, abortController.signal)) {+ response.content += chunk;+ this.onChange.fire();+ }+ } finally {+ this._abortControllers.delete(abortController);+ this.onChange.fire();+ }+ }++ isSending(): boolean {+ return this._abortControllers.size > 0;+ }++ abortSending() {+ for (const controller of this._abortControllers)+ controller.abort();+ this._abortControllers.clear();+ this.onChange.fire();+ }++ isEmpty() {+ return this.history.length < 2;+ }+}+++const llmContext = React.createContext<LLMChat | undefined>(undefined);++export function LLMProvider({ children }: React.PropsWithChildren<{}>) {+ const cookiePairs = useCookies();+ const chat = React.useMemo(() => {+ const cookies = Object.fromEntries(cookiePairs);+ if (cookies.openai_api_key)+ return new LLMChat(new OpenAI(cookies.openai_api_key, cookies.openai_base_url));+ if (cookies.anthropic_api_key)+ return new LLMChat(new Anthropic(cookies.anthropic_api_key, cookies.anthropic_base_url));+ }, [cookiePairs]);+ return <llmContext.Provider value={chat}>{children}</llmContext.Provider>;+}++export function useLLMChat() {+ const chat = React.useContext(llmContext);+ if (!chat)+ throw new Error('No LLM chat available, make sure theres a LLMProvider above');+ return chat;+}++export function useIsLLMAvailable() {+ return !!React.useContext(llmContext);+}++export function useLLMConversation(id: string) {+ const conversation = useLLMChat().getConversation(id);+ if (!conversation)+ throw new Error('No conversation found for id: ' + id);+ const [history, setHistory] = React.useState(conversation.history);+ React.useEffect(() => {+ function update() {+ setHistory([...conversation!.history]);+ }+ update();+ const subscription = conversation.onChange.event(update);+ return subscription.dispose;+ }, [conversation]);++ return [history, conversation] as const;+}diff --git a/packages/trace-viewer/src/ui/uiModeView.tsx b/packages/trace-viewer/src/ui/uiModeView.tsxindex 8e27ed0137..441163c5d9 100644--- a/packages/trace-viewer/src/ui/uiModeView.tsx+++ b/packages/trace-viewer/src/ui/uiModeView.tsx@@ -38,6 +38,7 @@ import { TraceView } from './uiModeTraceView';import { SettingsView } from './settingsView';import { DefaultSettingsView } from './defaultSettingsView';import { GitCommitInfoProvider } from './errorsTab';+import { LLMProvider } from './llm';let xtermSize = { cols: 80, rows: 24 };const xtermDataSource: XtermDataSource = {@@ -398,7 +399,7 @@ export const UIModeView: React.FC<{}> = ({});}, [closeInstallDialog, testServerConnection]);- return <div className='vbox ui-mode'>+ return <LLMProvider><div className='vbox ui-mode'>{!hasBrowsers && <dialog ref={dialogRef}><div className='title'><span className='codicon codicon-lightbulb'></span>Install browsers</div><div className='body'>@@ -527,5 +528,5 @@ export const UIModeView: React.FC<{}> = ({</div>}/>- </div>;+ </div></LLMProvider>;};diff --git a/packages/trace-viewer/src/ui/workbench.tsx b/packages/trace-viewer/src/ui/workbench.tsxindex 25d01098ed..34f27d65cb 100644--- a/packages/trace-viewer/src/ui/workbench.tsx+++ b/packages/trace-viewer/src/ui/workbench.tsx@@ -199,7 +199,7 @@ export const Workbench: React.FunctionComponent<{elsesetRevealedError(error);selectPropertiesTab('source');- }} actions={model?.actions ?? []} />+ }} actions={model?.actions ?? []} wallTime={model?.wallTime ?? 0} />};// Fallback location w/o action stands for file / test.diff --git a/packages/trace-viewer/src/ui/workbenchLoader.tsx b/packages/trace-viewer/src/ui/workbenchLoader.tsxindex 9f71eb1142..9a90523f9a 100644--- a/packages/trace-viewer/src/ui/workbenchLoader.tsx+++ b/packages/trace-viewer/src/ui/workbenchLoader.tsx@@ -21,6 +21,7 @@ import './workbenchLoader.css';import { Workbench } from './workbench';import { TestServerConnection, WebSocketTestServerTransport } from '@testIsomorphic/testServerConnection';import { SettingsToolbarButton } from './settingsToolbarButton';+import { LLMProvider } from './llm';export const WorkbenchLoader: React.FunctionComponent<{}> = () => {@@ -165,7 +166,9 @@ export const WorkbenchLoader: React.FunctionComponent<{<div className='progress'><div className='inner-progress' style={{ width: progress.total ? (100 * progress.done / progress.total) + '%' : 0 }}></div></div>- <Workbench model={model} inert={showFileUploadDropArea} />+ <LLMProvider>+ <Workbench model={model} inert={showFileUploadDropArea} />+ </LLMProvider>{fileForLocalModeError && <div className='drop-target'><div>Trace Viewer uses Service Workers to show traces. To view trace:</div><div style={{ paddingTop: 20 }}>diff --git a/packages/web/src/uiUtils.ts b/packages/web/src/uiUtils.tsindex a0b7a59c36..cded19af20 100644--- a/packages/web/src/uiUtils.ts+++ b/packages/web/src/uiUtils.ts@@ -248,3 +248,13 @@ export function useFlash(): [boolean, EffectCallback] {}, [setFlash]);return [flash, trigger];}++export function useCookies() {+ const cookies = React.useMemo(() => {+ return document.cookie.split('; ').filter(v => v.includes('=')).map(kv => {+ const separator = kv.indexOf('=');+ return [kv.substring(0, separator), kv.substring(separator + 1)];+ });+ }, []);+ return cookies;+}diff --git a/tests/playwright-test/ui-mode-llm.spec.ts b/tests/playwright-test/ui-mode-llm.spec.tsnew file mode 100644index 0000000000..8c7b0447ec--- /dev/null+++ b/tests/playwright-test/ui-mode-llm.spec.ts@@ -0,0 +1,99 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect, retries } from './ui-mode-fixtures';++test.describe.configure({ mode: 'parallel', retries });++test('openai', async ({ runUITest, server }) => {+ server.setRoute('/v1/chat/completions', async (req, res) => {+ res.setHeader('Access-Control-Allow-Origin', '*');+ res.setHeader('Access-Control-Allow-Headers', '*');+ if (req.method === 'OPTIONS')+ return res.end();++ expect(req.headers.authorization).toBe('Bearer fake-key');+ expect((await req.postBody).toString()).toContain(`- button \\"Submit\\"`);+ const event = {+ object: 'chat.completion.chunk',+ choices: [{ delta: { content: 'This is a mock response' } }]+ };+ res.setHeader('Content-Type', 'text/event-stream');+ res.write(`data: ${JSON.stringify(event)}\n\n`);+ });++ const { page } = await runUITest({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('trace test', async ({ page }) => {+ await page.setContent('<button>Submit</button>');+ expect(1).toBe(2);+ });+ `,+ }, {+ OPENAI_API_KEY: 'fake-key',+ OPENAI_BASE_URL: server.PREFIX,+ });++ await page.getByTitle('Run all').click();+ await page.getByText('Errors', { exact: true }).click();+ await page.getByRole('button', { name: 'Fix with AI' }).click();+ await expect(page.getByRole('tabpanel', { name: 'Errors' })).toMatchAriaSnapshot(`+ - tabpanel "Errors":+ - text: Help me with the error above. Take the page snapshot into account.+ - text: This is a mock response+ `);+});++test('anthropic', async ({ runUITest, server }) => {+ server.setRoute('/v1/messages', async (req, res) => {+ res.setHeader('Access-Control-Allow-Origin', '*');+ res.setHeader('Access-Control-Allow-Headers', '*');+ if (req.method === 'OPTIONS')+ return res.end();++ expect(req.headers['x-api-key']).toBe('fake-key');+ expect((await req.postBody).toString()).toContain(`- button \\"Submit\\"`);+ const event = {+ type: 'content_block_delta',+ delta: { text: 'This is a mock response' },+ };+ res.setHeader('Content-Type', 'text/event-stream');+ res.write(`data: ${JSON.stringify(event)}\n\n`);+ });++ const { page } = await runUITest({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('trace test', async ({ page }) => {+ await page.setContent('<button>Submit</button>');+ expect(1).toBe(2);+ });+ `,+ }, {+ ANTHROPIC_API_KEY: 'fake-key',+ ANTHROPIC_BASE_URL: server.PREFIX,+ });++ await page.getByTitle('Run all').click();+ await page.getByText('Errors', { exact: true }).click();+ await page.getByRole('button', { name: 'Fix with AI' }).click();+ await expect(page.getByRole('tabpanel', { name: 'Errors' })).toMatchAriaSnapshot(`+ - tabpanel "Errors":+ - text: Help me with the error above. Take the page snapshot into account.+ - text: This is a mock response+ `);+});