Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🎬 test: Record-Once/Replay-Forever Model Fixtures for Mock E2E (#15210)
Danny Avila1489623fa3aa
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.jsindex 6c1b6a6136..42195a0c3e 100644--- a/api/server/controllers/agents/client.js+++ b/api/server/controllers/agents/client.js@@ -3600,6 +3600,9 @@ class AgentClient extends BaseClient {(activityLabel ? createAssistantPhaseStampingHandlers(offsetHandlers) : offsetHandlers);const createRunPromise = createRun({agents,+ // Conversation-stable identity for the e2e run hook; a resumed run+ // carries no messages, so history cannot identify the conversation.+ conversationId: this.conversationId,messages,modelCallbacks: [modelBoundCallback],// This controller implements the full HITL pause/resume lifecycle (handleRunInterrupt@@ -4070,6 +4073,7 @@ class AgentClient extends BaseClient {(activityLabel ? createAssistantPhaseStampingHandlers(offsetHandlers) : offsetHandlers);run = await createRun({agents,+ conversationId: this.conversationId,modelCallbacks: [modelBoundCallback],// State (messages, tool calls) is rehydrated from the checkpoint by// run.resume; createRun only needs the agents to rebuild the graph.diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yamlindex 07dda8b569..92efa48463 100644--- a/e2e/config/librechat.e2e.yaml+++ b/e2e/config/librechat.e2e.yaml@@ -92,6 +92,10 @@ endpoints:- module: e2e/setup/tool-approval-hook.jsmatcher: ^approval_probe_mcp_e2e-memory$custom:+ # Substituted with a REAL provider endpoint only when the mock config runs+ # in model-fixture record mode (E2E_MODEL_FIXTURES=record); a comment+ # otherwise, so the credential-free profile never gains a live endpoint.+ # __E2E_MODEL_RECORD_PROVIDER__- name: 'Mock Provider A'apiKey: 'e2e-mock-key-a'baseURL: 'http://127.0.0.1:8889/v1'@@ -184,6 +188,7 @@ modelSpecs:# Surfaces the endpoints menu (modelSelect defaults on when addedEndpoints is# set) limited to entries that don't collide with the spec labels above.addedEndpoints:+ # __E2E_MODEL_RECORD_ADDED_ENDPOINT__- 'Mock Provider C'- 'Mock Provider D'- 'Mock Provider E'diff --git a/e2e/fixtures/model-replay/deepseek-tool-call.jsonl b/e2e/fixtures/model-replay/deepseek-tool-call.jsonlnew file mode 100644index 0000000000..0ede2d819c--- /dev/null+++ b/e2e/fixtures/model-replay/deepseek-tool-call.jsonl@@ -0,0 +1,40 @@+{"type":"meta","name":"deepseek-tool-call","recordedAt":"2026-08-25T22:23:39.533Z"}+{"type":"invocation","index":0,"userText":"Call the remember_fact tool with fact set to \"the replay lane records tool calls\", then reply with exactly the text the tool returned and nothing else."}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"id":"call_00_MvSzx7ML4G6RE6Twg4cr3997","type":"function","function":{"name":"remember_fact_mcp_e2e-memory","arguments":""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"name":"remember_fact_mcp_e2e-memory","args":"","id":"call_00_MvSzx7ML4G6RE6Twg4cr3997","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"{"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"{","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"fact"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"fact","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":": "}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":": ","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"the"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"the","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" replay"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" replay","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" lane"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" lane","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" records"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" records","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" tool"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" tool","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" calls"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" calls","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"}","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"finish_reason":"tool_calls","system_fingerprint":"a26a7955944dc5c60445bff77fac9c8e","model_name":"deepseek-v4-flash","model_provider":"openai","usage":{"prompt_tokens":517,"completion_tokens":56,"total_tokens":573,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":5}},"tool_call_chunks":[],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":517,"completion_tokens":56,"total_tokens":573,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":5}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":517,"output_tokens":56,"total_tokens":573,"input_token_details":{"cache_read":512}},"id":"run-01a03b05-a093-744d-a6c6-dbf53d88db4a"}}+{"type":"end","invocation":0,"text":""}+{"type":"invocation","index":1,"userText":"Call the remember_fact tool with fact set to \"the replay lane records tool calls\", then reply with exactly the text the tool returned and nothing else."}+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":"E","message":{"content":"E","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":"2","message":{"content":"2","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":"E","message":{"content":"E","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" M","message":{"content":" M","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":"CP","message":{"content":"CP","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" memory","message":{"content":" memory","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" noted","message":{"content":" noted","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":":","message":{"content":":","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" the","message":{"content":" the","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" replay","message":{"content":" replay","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" lane","message":{"content":" lane","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" records","message":{"content":" records","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" tool","message":{"content":" tool","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":" calls","message":{"content":" calls","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"finish_reason":"stop","system_fingerprint":"a26a7955944dc5c60445bff77fac9c8e","model_name":"deepseek-v4-flash","model_provider":"openai","usage":{"prompt_tokens":599,"completion_tokens":14,"total_tokens":613,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":87}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":599,"completion_tokens":14,"total_tokens":613,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":87}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":599,"output_tokens":14,"total_tokens":613,"input_token_details":{"cache_read":512}},"id":"run-01a03b05-a445-728f-821b-b6b325e263e9"}}+{"type":"end","invocation":1,"text":"E2E MCP memory noted: the replay lane records tool calls"}diff --git a/e2e/fixtures/model-replay/deepseek-two-turn.jsonl b/e2e/fixtures/model-replay/deepseek-two-turn.jsonlnew file mode 100644index 0000000000..056d14e4ac--- /dev/null+++ b/e2e/fixtures/model-replay/deepseek-two-turn.jsonl@@ -0,0 +1,52 @@+{"type":"meta","name":"deepseek-two-turn","recordedAt":"2026-08-25T17:44:11.310Z"}+{"type":"invocation","index":0,"userText":"Name the two prime numbers between 20 and 30, comma separated, and nothing else."}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}+{"type":"chunk","invocation":0,"text":"23","message":{"content":"23","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}+{"type":"chunk","invocation":0,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}+{"type":"chunk","invocation":0,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}+{"type":"chunk","invocation":0,"text":"29","message":{"content":"29","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{"prompt_tokens":23,"completion_tokens":4,"total_tokens":27,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":23}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":23,"completion_tokens":4,"total_tokens":27,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":23}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":23,"output_tokens":4,"total_tokens":27,"input_token_details":{"cache_read":0}}}}+{"type":"end","invocation":0,"text":"23, 29"}+{"type":"invocation","index":1,"userText":"In two short sentences, explain why the sum of those two primes is an even number. Begin with the word \"Because\"."}+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"Because","message":{"content":"Because","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" any","message":{"content":" any","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" prime","message":{"content":" prime","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" number","message":{"content":" number","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" greater","message":{"content":" greater","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" than","message":{"content":" than","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"2","message":{"content":"2","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" odd","message":{"content":" odd","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" and","message":{"content":" and","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" the","message":{"content":" the","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" sum","message":{"content":" sum","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" of","message":{"content":" of","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" two","message":{"content":" two","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" odd","message":{"content":" odd","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" numbers","message":{"content":" numbers","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" always","message":{"content":" always","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" even","message":{"content":" even","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":".","message":{"content":".","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" Therefore","message":{"content":" Therefore","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"23","message":{"content":"23","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" plus","message":{"content":" plus","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"29","message":{"content":"29","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" equals","message":{"content":" equals","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"52","message":{"content":"52","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" which","message":{"content":" which","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":" even","message":{"content":" even","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":".","message":{"content":".","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{"prompt_tokens":56,"completion_tokens":37,"total_tokens":93,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":56}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":56,"completion_tokens":37,"total_tokens":93,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":56}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":56,"output_tokens":37,"total_tokens":93,"input_token_details":{"cache_read":0}}}}+{"type":"end","invocation":1,"text":"Because any prime number greater than 2 is odd, and the sum of two odd numbers is always even. Therefore, 23 plus 29 equals 52, which is even."}diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.tsindex 694061c1b2..0787f72fc0 100644--- a/e2e/playwright.config.mock.ts+++ b/e2e/playwright.config.mock.ts@@ -26,6 +26,67 @@ const labelServerPath = path.resolve(rootPath, 'e2e/setup/fake-label-server.js')* `writeRuntimeMockConfig` substitutes any override into the generated copy. */const LABEL_PORT = process.env.E2E_LABEL_PORT || '8889';const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js');+/** Model-fixture record mode: the run hook taps the REAL provider stream into a+ * replayable fixture instead of overriding the model (e2e/setup/model-replay.js). */+const modelFixtureRecording = process.env.E2E_MODEL_FIXTURES === 'record';+const recordModelHookPath = path.resolve(rootPath, 'e2e/setup/record-model.js');+const recordProviderBaseURL =+ process.env.E2E_RECORD_PROVIDER_BASE_URL || 'https://api.deepseek.com/v1';+const recordProviderModel = process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat';+if (modelFixtureRecording && !process.env.E2E_RECORD_PROVIDER_API_KEY) {+ throw new Error('E2E_MODEL_FIXTURES=record requires E2E_RECORD_PROVIDER_API_KEY');+}+/**+ * Each fixture belongs to exactly one spec, and a spec records only its own.+ * Accepting an arbitrary name would leave a second fixture beside the+ * committed one carrying the same prompts, and the server-side ambiguity+ * check would then refuse to bind either — a successful recording run would+ * disable the keyless lane.+ */+const RECORDABLE_FIXTURES = ['deepseek-two-turn', 'deepseek-tool-call'];+if (modelFixtureRecording && !process.env.E2E_MODEL_FIXTURE_NAME) {+ throw new Error('E2E_MODEL_FIXTURES=record requires E2E_MODEL_FIXTURE_NAME');+}+if (+ modelFixtureRecording &&+ !RECORDABLE_FIXTURES.includes(process.env.E2E_MODEL_FIXTURE_NAME ?? '')+) {+ throw new Error(+ `E2E_MODEL_FIXTURE_NAME must be one of ${RECORDABLE_FIXTURES.join(', ')}; ` ++ `received ${process.env.E2E_MODEL_FIXTURE_NAME}`,+ );+}+/**+ * Playwright documents `-c` as an alias for `--config`, so both spellings are+ * parsed — recognising only the long form would let the short one slip past.+ *+ * Derived configs (`playwright.config.redis.ts`, `.mermaid.ts`) spread this+ * config and then replace `testMatch`, discarding the record-mode restriction+ * below — their specs would reach the paid provider and rewrite the selected+ * fixture. The restriction cannot be enforced through a value a consumer can+ * overwrite, so record mode refuses any config but this one.+ */+if (modelFixtureRecording) {+ /** Only the process that parsed the CLI carries `--config`; Playwright+ * workers do not, and must not be judged on an argument they never saw. */+ const configFlagIndex = process.argv.findIndex(+ (arg) =>+ arg === '--config' || arg === '-c' || arg.startsWith('--config=') || arg.startsWith('-c='),+ );+ const configFlag = configFlagIndex === -1 ? undefined : process.argv[configFlagIndex];+ let configPath: string | undefined;+ if (configFlag?.includes('=')) {+ configPath = configFlag.slice(configFlag.indexOf('=') + 1);+ } else if (configFlag) {+ configPath = process.argv[configFlagIndex + 1];+ }+ if (configPath && !/playwright\.config\.mock\.ts$/.test(configPath)) {+ throw new Error(+ `E2E_MODEL_FIXTURES=record only runs under playwright.config.mock.ts, not ${configPath}; ` ++ 'derived configs replace testMatch and would send their specs to the real provider',+ );+ }+}const assistantsServerPath = path.resolve(rootPath, 'e2e/setup/fake-assistants-server.js');const ASSISTANTS_PORT = process.env.E2E_ASSISTANTS_PORT || '8890';const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');@@ -61,8 +122,15 @@ const baseEnv = {...getLocalE2EEnv(),CONFIG_PATH: configPath,DEPLOYMENT_SKILLS_DIR: deploymentSkillsPath,- /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */- LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath,+ /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model —+ * or, in model-fixture record mode, to tap the real provider stream. */+ LIBRECHAT_TEST_RUN_HOOK: modelFixtureRecording ? recordModelHookPath : fakeModelHookPath,+ ...(modelFixtureRecording+ ? {+ E2E_MODEL_FIXTURE_NAME: process.env.E2E_MODEL_FIXTURE_NAME ?? '',+ E2E_RECORD_PROVIDER_API_KEY: process.env.E2E_RECORD_PROVIDER_API_KEY ?? '',+ }+ : {}),...(enableDynamicMcp ? { E2E_MCP_LIST_CHANGED: 'true', E2E_MCP_STATE_PATH: MCP_STATE_PATH } : {}),/** The Assistants runtime uses the OpenAI SDK directly, outside the agents run hook. */ASSISTANTS_API_KEY: 'e2e-mock-assistants-key',@@ -117,7 +185,27 @@ function writeRuntimeMockConfig() {].join('\n'),}: { allowedDomain: '', stdioEnv: '', networkServers: '' };+ const recordProviderBlock = modelFixtureRecording+ ? [+ `- name: 'Replay Record Provider'`,+ ` apiKey: '\${E2E_RECORD_PROVIDER_API_KEY}'`,+ ` baseURL: '${recordProviderBaseURL}'`,+ ' models:',+ ' default:',+ ` - '${recordProviderModel}'`,+ ' fetch: false',+ ' titleConvo: false',+ ` modelDisplayLabel: 'Replay Record Provider'`,+ ].join('\n ')+ : '# __E2E_MODEL_RECORD_PROVIDER__';config = config+ .replace('# __E2E_MODEL_RECORD_PROVIDER__', recordProviderBlock)+ .replace(+ '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__',+ modelFixtureRecording+ ? `- 'Replay Record Provider'`+ : '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__',+ ).replace('# __E2E_DYNAMIC_MCP_ALLOWED_DOMAIN__', dynamicMcpConfig.allowedDomain).replace('# __E2E_DYNAMIC_MCP_STDIO_ENV__', dynamicMcpConfig.stdioEnv).replace('# __E2E_DYNAMIC_MCP_NETWORK_SERVERS__', dynamicMcpConfig.networkServers);@@ -177,6 +265,12 @@ export default defineConfig({globalSetup: require.resolve('./setup/global-setup'),globalTeardown: require.resolve('./setup/global-teardown.mock'),testDir: 'specs/mock/',+ /** Record mode swaps the fake model for a real provider, so it must never+ * run the whole mock suite: every spec's prompts would reach the paid+ * endpoint, and each fresh conversation would truncate and rewrite the one+ * selected fixture, leaving whichever scenario ran last. Without this an+ * unfiltered entry point (`npm run e2e:mock`) does exactly that. */+ ...(modelFixtureRecording ? { testMatch: /model-replay[a-z-]*\.spec\.ts$/ } : {}),outputDir: 'specs/.test-results',fullyParallel: true,forbidOnly: !!process.env.CI,diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.jsindex a6e69ac67d..26e31c63ec 100644--- a/e2e/setup/fake-model.js+++ b/e2e/setup/fake-model.js@@ -12,6 +12,7 @@const { FakeChatModel } = require('@librechat/agents');const { ChatGenerationChunk } = require('@langchain/core/outputs');const { AIMessageChunk } = require('@langchain/core/messages');+const { tryBindReplay } = require('./model-replay');const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong';const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;@@ -2393,6 +2394,21 @@ module.exports = function fakeModelHook(run, context) {}const text = getLatestUserText(context?.messages);+ /** Recorded-session replay outranks marker routing: a conversation whose+ * prompt matches a fixture's next recorded invocation streams that recording+ * through the real pipeline instead of a scripted mock response. */+ if (+ tryBindReplay({+ graph,+ text,+ agents: context?.agents,+ messages: context?.messages,+ conversationId: context?.conversationId,+ modelCallbacks: context?.modelCallbacks,+ })+ ) {+ return;+ }const toolNames = collectToolNames(context?.agents);const handoffScript = parseHandoffScript(text);const {diff --git a/e2e/setup/model-replay.js b/e2e/setup/model-replay.jsnew file mode 100644index 0000000000..50ca0eab4e--- /dev/null+++ b/e2e/setup/model-replay.js@@ -0,0 +1,732 @@+/**+ * Record-once/replay-forever model fixtures for the mock e2e harness.+ *+ * Record mode (`E2E_MODEL_FIXTURES=record` + `E2E_MODEL_FIXTURE_NAME=<name>`):+ * `record-model.js` replaces the fake-model run hook; instead of overriding the+ * graph's model it appends a LangChain callback handler to every agent+ * context's `clientOptions.callbacks`, so the REAL provider model carries the+ * recorder. Each model invocation's streamed `ChatGenerationChunk`s are+ * serialized to `e2e/fixtures/model-replay/<name>.jsonl` exactly as the+ * provider emitted them (text deltas, tool_call_chunks, reasoning+ * additional_kwargs, usage_metadata). Only the invocation's latest human text+ * is recorded for binding — system prompts and tool schemas never enter the+ * fixture.+ *+ * Replay mode (default, keyless): `fake-model.js` consults `tryBindReplay`+ * before its marker routing. A conversation binds to a fixture when its latest+ * user text equals the fixture's next unconsumed invocation's recorded user+ * text. The replaying model is not hand-assigned: `ReplayChatModel` is+ * registered as the SDK provider `librechat-e2e-replay` via the agents+ * package's `registerProvider`, and the bound instance is constructed through+ * the SDK's own `initializeModel` — registry lookup, constructor+ * `clientOptions` (carrying the model-bound callbacks the way+ * `withModelCallbacks` does), and real `bindTools` over the run's tools — so+ * the recorded chunks stream through the same SDK machinery a live provider+ * uses: createRun → registered provider model → graph → SSE → persistence.+ * Every invocation re-checks its prompt against the recording, an invocation+ * past the end of the script throws (over-consumption fails loud in the+ * turn), and a per-fixture consumption ledger under+ * `e2e/specs/.test-results/model-replay/` lets specs assert at teardown that+ * every recorded invocation and chunk was drained (under-consumption fails the+ * spec, not silently).+ *+ * Constraint carried over from the recording model: one live binding per+ * fixture per server process — scenarios replaying the same fixture must not+ * run concurrently.+ */+const fs = require('fs');+const path = require('path');+const { FakeChatModel, registerProvider, initializeModel } = require('@librechat/agents');+const { ChatGenerationChunk } = require('@langchain/core/outputs');+const { AIMessageChunk } = require('@langchain/core/messages');++const FIXTURES_DIR = path.resolve(__dirname, '../fixtures/model-replay');+const LEDGER_DIR = path.resolve(__dirname, '../specs/.test-results/model-replay');+const RECORDER_HANDLER_NAME = 'librechat-e2e-model-recorder';+const SUMMARIZATION_GUARD_NAME = 'librechat-e2e-summarization-guard';+const REPLAY_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;++function extractText(content) {+ if (typeof content === 'string') {+ return content;+ }+ if (!Array.isArray(content)) {+ return '';+ }+ const parts = [];+ for (const part of content) {+ if (typeof part === 'string') {+ parts.push(part);+ } else if (part && typeof part.text === 'string') {+ parts.push(part.text);+ }+ }+ return parts.join('');+}++function messageType(message) {+ if (typeof message?.getType === 'function') {+ return message.getType();+ }+ if (typeof message?._getType === 'function') {+ return message._getType();+ }+ return message?.role;+}++/** Every human message's text, oldest first. */+function humanTexts(messages) {+ if (!Array.isArray(messages)) {+ return [];+ }+ const texts = [];+ for (const message of messages) {+ const type = messageType(message);+ if (type === 'human' || type === 'user') {+ texts.push(extractText(message.content));+ }+ }+ return texts;+}++/** The latest human message's text — the binding and prompt-check key. */+function latestHumanText(messages) {+ const texts = humanTexts(messages);+ return texts.length > 0 ? texts[texts.length - 1] : '';+}++/**+ * The distinct user turns a fixture records, in order. A turn that calls a+ * tool spans several model invocations under one prompt, so the invocation+ * sequence is not the turn sequence and only this collapsed view can be+ * compared against a conversation's human messages.+ */+function fixtureTurnTexts(invocations) {+ const turns = [];+ for (const invocation of invocations) {+ if (turns[turns.length - 1] !== invocation.userText) {+ turns.push(invocation.userText);+ }+ }+ return turns;+}++/**+ * Whether this conversation is the one that already drove the fixture: its+ * human turns open with exactly the fixture's recorded turns, in order. A+ * consumed binding has to be retained for such a conversation, or an extra+ * user turn would find no next invocation, fall through to ordinary+ * fake-model routing, and be answered with a mock reply — leaving the+ * over-consumption guard unreached and the drained ledger still passing.+ */+function conversationDroveFixture(messages, fixture) {+ const texts = humanTexts(messages);+ const turns = fixture.turns;+ if (texts.length <= turns.length) {+ return false;+ }+ return turns.every((turn, index) => turn === texts[index]);+}++function jsonClone(value) {+ if (value == null) {+ return undefined;+ }+ try {+ return JSON.parse(JSON.stringify(value));+ } catch {+ return undefined;+ }+}++/** Minimal AIMessageChunk projection that reconstructs the streamed message. */+function serializeChunk(chunk, token) {+ const message = chunk?.message;+ const serialized = { text: chunk?.text ?? token ?? '' };+ if (message) {+ serialized.message = {+ content: jsonClone(message.content) ?? '',+ additional_kwargs: jsonClone(message.additional_kwargs),+ response_metadata: jsonClone(message.response_metadata),+ tool_call_chunks: jsonClone(message.tool_call_chunks),+ usage_metadata: jsonClone(message.usage_metadata),+ id: typeof message.id === 'string' ? message.id : undefined,+ };+ }+ return serialized;+}++function deserializeChunk(serialized) {+ const recorded = serialized.message;+ const message = new AIMessageChunk({+ content: recorded?.content ?? serialized.text ?? '',+ additional_kwargs: recorded?.additional_kwargs ?? {},+ response_metadata: recorded?.response_metadata ?? {},+ tool_call_chunks: recorded?.tool_call_chunks ?? [],+ usage_metadata: recorded?.usage_metadata,+ id: recorded?.id,+ });+ return new ChatGenerationChunk({ text: serialized.text ?? '', message });+}++/* ------------------------------- recording ------------------------------- */++const recordingState = {+ initialized: false,+ fixturePath: undefined,+ invocationCounter: 0,+ conversationId: undefined,+ /** Bumped on every (re)start so handlers left on a superseded graph can be+ * told apart from the current attempt's. */+ generation: 0,+ runIdToInvocation: new Map(),+};++function appendFixtureLine(entry) {+ fs.appendFileSync(recordingState.fixturePath, `${JSON.stringify(entry)}\n`);+}++function initializeRecording(fixtureName) {+ fs.mkdirSync(FIXTURES_DIR, { recursive: true });+ recordingState.fixturePath = path.join(FIXTURES_DIR, `${fixtureName}.jsonl`);+ fs.writeFileSync(recordingState.fixturePath, '');+ appendFixtureLine({+ type: 'meta',+ name: fixtureName,+ recordedAt: new Date().toISOString(),+ });+ recordingState.initialized = true;+ recordingState.invocationCounter = 0;+ recordingState.conversationId = undefined;+ recordingState.generation += 1;+ recordingState.runIdToInvocation.clear();+ console.log(`[e2e model-replay] recording fixture ${recordingState.fixturePath}`);+}++/**+ * The recorder's state is process-global and the web server outlives a+ * Playwright retry, so a failed attempt that already recorded invocations+ * would otherwise leave the counter advanced: the retry appends 2/3 after+ * 0/1 (or keeps a previous attempt's `error` line) and the fixture is+ * unusable for replay.+ *+ * A new attempt is a new conversation. Identity comes from `conversationId`+ * rather than from the prompt or the history: a turn that calls a tool+ * invokes the model again under the same latest human message, and a resumed+ * run after a tool-approval pause rebuilds `createRun` with no messages at+ * all because state is rehydrated from the checkpoint. Both would look like+ * fresh attempts to any text- or history-based rule, and truncate the fixture+ * mid-turn.+ */+function isConversationStart(messages) {+ return humanTexts(messages).length <= 1;+}++function startsNewRecording(conversationId, messages) {+ if (recordingState.invocationCounter === 0) {+ return false;+ }+ if (conversationId != null && recordingState.conversationId != null) {+ return recordingState.conversationId !== conversationId;+ }+ return isConversationStart(messages);+}++/**+ * Handlers are stamped with the recording generation they were installed for.+ * A failed attempt can still have a provider call in flight when the retry+ * resets the recording, and its graph keeps this handler: without the stamp+ * that stale call would allocate an invocation index from the new attempt's+ * counter, or append an `error` entry whose mapping was cleared, corrupting+ * the freshly reset fixture.+ */+function createRecorderHandler() {+ const generation = recordingState.generation;+ const superseded = () => generation !== recordingState.generation;+ return {+ name: RECORDER_HANDLER_NAME,+ generation,+ /** Callbacks must settle before the model call resolves, or the `end`+ * line races the durable-completion barrier the recording spec waits on+ * (the same contract ModelBoundChatModelCallback declares). */+ awaitHandlers: true,+ raiseError: true,+ handleChatModelStart(_llm, messageBatches, runId) {+ if (superseded()) {+ return;+ }+ const index = recordingState.invocationCounter++;+ recordingState.runIdToInvocation.set(runId, index);+ appendFixtureLine({+ type: 'invocation',+ index,+ userText: latestHumanText(messageBatches?.[0]),+ });+ },+ handleLLMNewToken(token, _idx, runId, _parentRunId, _tags, fields) {+ const invocation = recordingState.runIdToInvocation.get(runId);+ if (superseded() || invocation == null) {+ return;+ }+ appendFixtureLine({+ type: 'chunk',+ invocation,+ ...serializeChunk(fields?.chunk, token),+ });+ },+ handleLLMEnd(output, runId) {+ const invocation = recordingState.runIdToInvocation.get(runId);+ if (superseded() || invocation == null) {+ return;+ }+ recordingState.runIdToInvocation.delete(runId);+ const generation = output?.generations?.[0]?.[0];+ appendFixtureLine({+ type: 'end',+ invocation,+ text: generation?.text ?? extractText(generation?.message?.content),+ });+ },+ handleLLMError(error, runId) {+ const invocation = recordingState.runIdToInvocation.get(runId);+ recordingState.runIdToInvocation.delete(runId);+ if (superseded()) {+ return;+ }+ appendFixtureLine({+ type: 'error',+ invocation: invocation ?? null,+ message: error instanceof Error ? error.message : String(error),+ });+ },+ };+}++/**+ * Attach the recorder to every agent context's model client options. The model+ * is created per-invocation from `agentContext.clientOptions`, so appending a+ * callback here puts the recorder on the real provider stream without+ * replacing the model.+ */+function installRecorder({ graph, messages, conversationId }) {+ const fixtureName = process.env.E2E_MODEL_FIXTURE_NAME;+ if (!fixtureName) {+ console.warn('[e2e model-replay] E2E_MODEL_FIXTURE_NAME unset; not recording');+ return;+ }+ if (!recordingState.initialized || startsNewRecording(conversationId, messages)) {+ initializeRecording(fixtureName);+ }+ if (conversationId != null) {+ recordingState.conversationId = conversationId;+ }+ const contexts = graph?.agentContexts;+ if (!contexts || typeof contexts.values !== 'function') {+ console.warn('[e2e model-replay] graph.agentContexts unavailable; not recording');+ return;+ }+ for (const context of contexts.values()) {+ if (!context.clientOptions) {+ context.clientOptions = {};+ }+ attachRecorder(context.clientOptions);+ /** Summarization runs on its own model with its own callback list.+ * Recording those invocations without replaying them is worse than+ * ignoring them: they would take slots in the fixture sequence that+ * replay never consumes, so the next primary call would read the+ * summariser's chunks. Replay routes only the agent model+ * (`graph.overrideModel`) and subagents, so the honest boundary is to+ * refuse a recording the lane could not reproduce. */+ const summarizationParameters =+ context.summarizationConfig?.parameters ?? context.summarizationConfig?.config?.parameters;+ if (summarizationParameters) {+ attachSummarizationGuard(summarizationParameters);+ }+ }+}++/**+ * Fails a recording the moment the summarization model runs. Its invocations+ * would otherwise enter the fixture sequence unreplayable — see+ * `installRecorder`. Summarization fixtures need replay routing for that model+ * before they can be supported.+ */+function attachSummarizationGuard(options) {+ const handler = {+ name: SUMMARIZATION_GUARD_NAME,+ raiseError: true,+ awaitHandlers: true,+ handleChatModelStart() {+ throw new Error(+ '[e2e model-replay] summarization ran during recording, and replay cannot route the ' ++ 'summarization model — its invocations would desynchronise the fixture. Record a ' ++ 'scenario that stays under the context-pruning threshold.',+ );+ },+ };+ const existing = options.callbacks;+ if (Array.isArray(existing)) {+ if (!existing.some((entry) => entry?.name === SUMMARIZATION_GUARD_NAME)) {+ options.callbacks = [...existing, handler];+ }+ return;+ }+ if (existing == null) {+ options.callbacks = [handler];+ return;+ }+ if (+ typeof existing.addHandler === 'function' &&+ !existing.handlers?.some((entry) => entry?.name === SUMMARIZATION_GUARD_NAME)+ ) {+ existing.addHandler(handler);+ }+}++/** Append the recorder to a client-options object's callbacks, once. */+function attachRecorder(options) {+ /** Dedupe against the CURRENT generation only: a graph carried across a+ * recording restart still holds a superseded handler, which is inert, so+ * matching on name alone would leave that options object recording+ * nothing. */+ const isCurrent = (handler) =>+ handler?.name === RECORDER_HANDLER_NAME && handler.generation === recordingState.generation;+ const existing = options.callbacks;+ if (Array.isArray(existing)) {+ if (!existing.some(isCurrent)) {+ options.callbacks = [+ ...existing.filter((handler) => handler?.name !== RECORDER_HANDLER_NAME),+ createRecorderHandler(),+ ];+ }+ return;+ }+ if (existing == null) {+ options.callbacks = [createRecorderHandler()];+ return;+ }+ if (typeof existing.addHandler === 'function') {+ if (!existing.handlers?.some(isCurrent)) {+ existing.addHandler(createRecorderHandler());+ }+ }+}++/* -------------------------------- replay --------------------------------- */++/** name -> { meta, invocations: [{ userText, chunks: [], finalText }] } */+let fixtureRegistry;+/** name -> { cursor, chunksConsumed, ledger } */+const replayState = new Map();++function parseFixtureFile(filePath) {+ const name = path.basename(filePath, '.jsonl');+ const invocations = [];+ let meta = { name };+ const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);+ for (const line of lines) {+ const entry = JSON.parse(line);+ if (entry.type === 'meta') {+ meta = entry;+ } else if (entry.type === 'invocation') {+ invocations[entry.index] = { userText: entry.userText, chunks: [], finalText: '' };+ } else if (entry.type === 'chunk') {+ invocations[entry.invocation]?.chunks.push(entry);+ } else if (entry.type === 'end') {+ const invocation = invocations[entry.invocation];+ if (invocation) {+ invocation.finalText = entry.text ?? '';+ }+ } else if (entry.type === 'error') {+ throw new Error(+ `[e2e model-replay] fixture ${name} recorded a provider error (${entry.message}); ` ++ 're-record it before replaying',+ );+ }+ }+ const missing = invocations.findIndex((invocation) => invocation == null);+ if (missing !== -1) {+ throw new Error(`[e2e model-replay] fixture ${name} is missing invocation ${missing}`);+ }+ /** The file name is the fixture's identity — it is what `E2E_MODEL_FIXTURE_NAME`+ * selects, what the spec names, and what the ledger is written under. A+ * recorded `meta.name` is descriptive only: trusting it would let a renamed+ * or copied fixture collapse onto another's registry key and ledger. */+ return { meta: { ...meta, name }, invocations, turns: fixtureTurnTexts(invocations) };+}++function loadFixtureRegistry() {+ if (fixtureRegistry) {+ return fixtureRegistry;+ }+ fixtureRegistry = new Map();+ if (!fs.existsSync(FIXTURES_DIR)) {+ return fixtureRegistry;+ }+ for (const file of fs.readdirSync(FIXTURES_DIR)) {+ if (file.endsWith('.jsonl')) {+ const fixture = parseFixtureFile(path.join(FIXTURES_DIR, file));+ fixtureRegistry.set(fixture.meta.name, fixture);+ }+ }+ return fixtureRegistry;+}++function writeLedger(name) {+ const state = replayState.get(name);+ if (!state) {+ return;+ }+ fs.mkdirSync(LEDGER_DIR, { recursive: true });+ fs.writeFileSync(+ path.join(LEDGER_DIR, `${name}.json`),+ `${JSON.stringify({ fixture: name, ...state.ledger }, null, 2)}\n`,+ );+}++function freshReplayState(fixture) {+ return {+ cursor: 0,+ ledger: {+ invocationsTotal: fixture.invocations.length,+ chunksTotal: fixture.invocations.reduce(+ (total, invocation) => total + invocation.chunks.length,+ 0,+ ),+ invocationsConsumed: 0,+ chunksConsumed: 0,+ overruns: [],+ promptMismatches: [],+ },+ };+}++function getReplayState(fixture) {+ let state = replayState.get(fixture.meta.name);+ if (!state) {+ state = freshReplayState(fixture);+ replayState.set(fixture.meta.name, state);+ }+ return state;+}++/**+ * Start the fixture over for a new conversation. The web server outlives a+ * Playwright retry, so without this a consumed cursor would leave the retry+ * unable to bind its first prompt — it would fall through to marker routing+ * and fail deterministically, burning every configured retry. The ledger+ * resets with the cursor so the new attempt is judged on its own consumption+ * rather than accumulating the previous one's counts.+ */+function restartReplayState(fixture) {+ const state = freshReplayState(fixture);+ replayState.set(fixture.meta.name, state);+ return state;+}++const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));++const REPLAY_PROVIDER = 'librechat-e2e-replay';++/**+ * Constructed by the SDK's `initializeModel` through the provider registry, so+ * `clientOptions` is the full constructor contract: the fixture binding, the+ * shared cursor state, and the run's model-bound callbacks.+ */+class ReplayChatModel extends FakeChatModel {+ constructor(clientOptions = {}) {+ super({ responses: [''], sleep: 0, emitCustomEvent: false });+ this.fixture = clientOptions.fixture;+ this.state = clientOptions.state;+ this.boundToolNames = clientOptions.boundToolNames ?? [];+ if (clientOptions.callbacks) {+ this.callbacks = clientOptions.callbacks;+ }+ }++ /** Real SDK tool binding: returns a bound copy sharing the replay cursor. */+ bindTools(tools) {+ return new ReplayChatModel({+ fixture: this.fixture,+ state: this.state,+ callbacks: this.callbacks,+ boundToolNames: (tools ?? []).map((tool) => tool?.name ?? tool?.function?.name ?? 'unknown'),+ });+ }++ async *_streamResponseChunks(messages, _options, runManager) {+ const { fixture, state } = this;+ const invocation = fixture.invocations[state.cursor];+ if (!invocation) {+ state.ledger.overruns.push({+ at: new Date().toISOString(),+ userText: latestHumanText(messages),+ });+ writeLedger(fixture.meta.name);+ throw new Error(+ `[e2e model-replay] fixture ${fixture.meta.name} over-consumed: model invoked ` ++ `after all ${fixture.invocations.length} recorded invocations were drained`,+ );+ }+ /** A resumed run carries no human message — state is rehydrated from the+ * checkpoint — so there is no prompt to check against. Ownership already+ * established which conversation this is; enforcing the recorded prompt+ * here would reject every resume. Every real turn still gets checked. */+ const promptText = latestHumanText(messages);+ const carriesHumanTurn = humanTexts(messages).length > 0;+ if (carriesHumanTurn && promptText !== invocation.userText) {+ state.ledger.promptMismatches.push({+ invocation: state.cursor,+ expected: invocation.userText,+ received: promptText,+ });+ writeLedger(fixture.meta.name);+ throw new Error(+ `[e2e model-replay] fixture ${fixture.meta.name} invocation ${state.cursor} ` ++ `prompt mismatch: recorded ${JSON.stringify(invocation.userText)}, ` ++ `received ${JSON.stringify(promptText)}`,+ );+ }+ state.cursor += 1;+ for (const chunk of invocation.chunks) {+ await sleep(REPLAY_CHUNK_DELAY_MS);+ yield deserializeChunk(chunk);+ void runManager?.handleLLMNewToken(chunk.text ?? '');+ state.ledger.chunksConsumed += 1;+ }+ state.ledger.invocationsConsumed += 1;+ writeLedger(fixture.meta.name);+ }+}++let replayProviderRegistered = false;++function ensureReplayProviderRegistered() {+ if (replayProviderRegistered) {+ return;+ }+ try {+ registerProvider({ provider: REPLAY_PROVIDER, model: ReplayChatModel });+ } catch (error) {+ /** The SDK registry is globalThis-scoped while this guard is+ * module-scoped: a reloaded copy of this module finds the provider+ * already registered. The registered class is stateless (fixture and+ * cursor ride `clientOptions`), so any copy's registration serves all. */+ if (!String(error instanceof Error ? error.message : error).includes('already registered')) {+ throw error;+ }+ }+ replayProviderRegistered = true;+}++/**+ * Bind a conversation to a recorded fixture when its latest user text matches+ * the fixture's next unconsumed invocation. The replay model is built through+ * the SDK's registered-provider path (`registerProvider` ++ * `initializeModel`), including real `bindTools` over the run's tools.+ * Returns true when the graph's model was overridden with the replaying+ * model; false lets the fake-model marker routing proceed unchanged.+ */+/**+ * Decide how this run relates to a fixture.+ *+ * `own` — the conversation that claimed the fixture is back. Its cursor is+ * authoritative wherever it stands, including past the end, so an extra turn+ * reaches the over-consumption guard instead of falling through to the+ * scripted fake model, and a resumed run after a tool-approval pause keeps+ * replaying even though it arrives with no messages and no prompt text.+ *+ * `claim` — a different (or first) conversation opening the fixture: rewind+ * and take ownership. This is what a Playwright retry looks like.+ *+ * Anything else is refused, so an unrelated conversation can never continue+ * someone else's partly consumed script by happening to repeat a later prompt.+ */+function classifyBinding({ fixture, state, text, messages, conversationId }) {+ if (conversationId != null && state.conversationId != null) {+ if (state.conversationId === conversationId) {+ return 'own';+ }+ return fixture.turns[0] === text ? 'claim' : 'refuse';+ }+ /** Identity unavailable (an older `@librechat/api` does not supply it):+ * fall back to the text and history rules this lane used before. */+ if (state.cursor !== 0 && isConversationStart(messages)) {+ return fixture.turns[0] === text ? 'claim' : 'refuse';+ }+ if (fixture.invocations[state.cursor]?.userText === text) {+ return 'own';+ }+ if (state.cursor !== 0 && fixture.turns[0] === text) {+ return 'claim';+ }+ if (state.cursor >= fixture.invocations.length && conversationDroveFixture(messages, fixture)) {+ return 'own';+ }+ return 'refuse';+}++function tryBindReplay({ graph, agents, text, messages, conversationId, modelCallbacks }) {+ const registry = loadFixtureRegistry();+ const matches = [];+ for (const fixture of registry.values()) {+ let state = getReplayState(fixture);+ const binding = classifyBinding({ fixture, state, text, messages, conversationId });+ if (binding === 'refuse') {+ continue;+ }+ if (binding === 'claim') {+ state = restartReplayState(fixture);+ }+ if (conversationId != null) {+ state.conversationId = conversationId;+ }+ matches.push({ fixture, state });+ }++ if (matches.length === 0) {+ return false;+ }+ /** Binding order would otherwise follow filesystem enumeration, so a second+ * fixture sharing this prompt could silently redirect a scenario to the+ * wrong chunks and ledger. The spec's fixture choice never reaches this+ * server-side loop, so ambiguity has to fail rather than pick a winner. */+ if (matches.length > 1) {+ throw new Error(+ `[e2e model-replay] prompt matches ${matches.length} fixtures ` ++ `(${matches.map(({ fixture }) => fixture.meta.name).join(', ')}); ` ++ 'fixtures must not share a bindable prompt',+ );+ }++ const { fixture, state } = matches[0];+ ensureReplayProviderRegistered();+ const model = initializeModel({+ provider: REPLAY_PROVIDER,+ clientOptions: { fixture, state, callbacks: modelCallbacks },+ tools: agents?.[0]?.tools ?? [],+ });+ state.ledger.toolsBound = model.boundToolNames ?? [];+ graph.overrideModel = model;+ /** `graph.overrideModel` is not inherited by child executors, so a fixture+ * recording a subagent call — record mode captures child invocations, since+ * the recorder attaches to every agent context — would otherwise leave the+ * child on its configured provider: an underrun here, and a real provider+ * request in a lane that must stay keyless. */+ if (typeof graph.setSubagentModelOverride === 'function') {+ graph.setSubagentModelOverride(model);+ }+ writeLedger(fixture.meta.name);+ return true;+}++module.exports = {+ FIXTURES_DIR,+ LEDGER_DIR,+ installRecorder,+ tryBindReplay,+ latestHumanText,+ serializeChunk,+ deserializeChunk,+ parseFixtureFile,+};diff --git a/e2e/setup/record-model.js b/e2e/setup/record-model.jsnew file mode 100644index 0000000000..63ff76ab1d--- /dev/null+++ b/e2e/setup/record-model.js@@ -0,0 +1,23 @@+/**+ * Run hook for `E2E_MODEL_FIXTURES=record`: taps the REAL provider model's+ * stream instead of overriding it, writing each model invocation's chunks to+ * `e2e/fixtures/model-replay/$E2E_MODEL_FIXTURE_NAME.jsonl` for keyless replay+ * through `fake-model.js`. Set as `LIBRECHAT_TEST_RUN_HOOK` by the mock+ * Playwright config when recording; the server must be booted with a working+ * provider credential (`E2E_RECORD_PROVIDER_API_KEY`).+ */+const { installRecorder } = require('./model-replay');++/** @type {import('@librechat/api').TestRunHook} */+module.exports = function recordModelHook(run, context) {+ const graph = run?.Graph;+ if (!graph) {+ console.warn('[e2e model-replay] record hook: run.Graph unavailable');+ return;+ }+ installRecorder({+ graph,+ messages: context?.messages,+ conversationId: context?.conversationId,+ });+};diff --git a/e2e/specs/mock/model-replay-tools.spec.ts b/e2e/specs/mock/model-replay-tools.spec.tsnew file mode 100644index 0000000000..df7d1f1c50--- /dev/null+++ b/e2e/specs/mock/model-replay-tools.spec.ts@@ -0,0 +1,166 @@+import { expect, test } from '@playwright/test';+import type { Page } from '@playwright/test';+import type { TMessage } from 'librechat-data-provider';+import {+ assertFixtureConsumed,+ fixtureTurns,+ readReplayLedger,+ removeFixture,+} from './replay.helpers';+import {+ NEW_CHAT_PATH,+ fetchJson,+ getAccessToken,+ selectMockEndpoint,+ sendMessageAndWaitForCompletion,+} from './helpers';++/**+ * The tool-call half of the replay lane: one recorded turn in which the real+ * provider calls an MCP tool, the tool runs for real, and the model is invoked+ * a second time with its result.+ *+ * This is the shape a single prompt cannot express — one user turn spanning+ * several model invocations — so it is what proves the fixture format carries+ * `tool_call_chunks` and that replay advances through a turn's invocations+ * rather than binding once per prompt.+ *+ * Record (needs a real provider key):+ * E2E_MODEL_FIXTURES=record E2E_MODEL_FIXTURE_NAME=deepseek-tool-call \+ * E2E_RECORD_PROVIDER_API_KEY=<key> \+ * npx playwright test --config=e2e/playwright.config.mock.ts model-replay-tools+ *+ * Replay (default, keyless): the same drive steps against the committed+ * fixture, with the recorded tool call streamed back through the real graph so+ * the tool executes again.+ */+const COMMITTED_FIXTURE = 'deepseek-tool-call';+const RECORDING = process.env.E2E_MODEL_FIXTURES === 'record';+const RECORDING_THIS_FIXTURE =+ RECORDING && process.env.E2E_MODEL_FIXTURE_NAME === COMMITTED_FIXTURE;+const FIXTURE = COMMITTED_FIXTURE;+const RECORD_ENDPOINT = {+ label: 'Replay Record Provider',+ model: process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat',+};++const MCP_SERVER_TITLE = 'E2E Memory';+const TOOL_NAME = 'remember_fact';+/** MCP tools reach the model under a server-qualified name+ * (`remember_fact_mcp_e2e-memory`), and that qualification has changed before,+ * so assertions match the base name as a prefix rather than pinning the suffix. */+const namesTool = (names: string[]) => names.some((name) => name.startsWith(TOOL_NAME));+/** The MCP fixture echoes this back, so the tool result is deterministic. */+const FACT = 'the replay lane records tool calls';+const TOOL_PROMPT =+ `Call the ${TOOL_NAME} tool with fact set to "${FACT}", then reply with exactly the ` ++ 'text the tool returned and nothing else.';++/** Enable the ephemeral MCP server whose tools this turn calls. */+async function selectEphemeralMCP(page: Page) {+ await page.getByRole('button', { name: 'MCP Servers', exact: true }).click();+ const serverItem = page.getByRole('menuitemcheckbox', { name: new RegExp(MCP_SERVER_TITLE) });+ await expect(serverItem).toBeVisible();+ await serverItem.click();+ await expect(serverItem).toHaveAttribute('aria-checked', 'true');+ await page.keyboard.press('Escape');+ await expect(page.getByRole('button', { name: new RegExp(MCP_SERVER_TITLE) })).toBeVisible();+}++test.describe('recorded tool-call fixture replay', () => {+ test('a recorded tool call replays through the real tool node', async ({ page }) => {+ test.skip(+ RECORDING && !RECORDING_THIS_FIXTURE,+ `recording ${process.env.E2E_MODEL_FIXTURE_NAME}`,+ );+ test.setTimeout(180_000);+ const pageErrors: string[] = [];+ page.on('pageerror', (error) => pageErrors.push(error.message));++ await page.goto(NEW_CHAT_PATH, { timeout: 10_000 });+ if (RECORDING) {+ removeFixture(FIXTURE);+ await selectMockEndpoint(page, RECORD_ENDPOINT);+ }+ await selectEphemeralMCP(page);++ await sendMessageAndWaitForCompletion(page, TOOL_PROMPT, { timeout: 120_000 });++ const conversationId = /\/c\/([^/]+)/.exec(page.url())?.[1];+ expect(conversationId, 'conversation should have a persisted id').toBeTruthy();+ const token = await getAccessToken(page);+ const messages = await fetchJson<TMessage[]>(+ page,+ `/api/messages/${encodeURIComponent(conversationId as string)}`,+ token,+ );+ const assistant = messages.filter((message) => message.isCreatedByUser === false);+ expect(assistant).toHaveLength(1);+ /** The turn's durable proof that the tool ran: a persisted tool_call part+ * naming the tool, independent of whatever prose the model wrapped it in. */+ const toolCallParts = (assistant[0].content ?? []).filter((part) => part?.type === 'tool_call');+ expect(toolCallParts.length, 'the turn should persist a tool call').toBeGreaterThan(0);+ expect(JSON.stringify(toolCallParts)).toContain(TOOL_NAME);++ if (RECORDING) {+ await expect+ .poll(+ () => {+ try {+ const settled = fixtureTurns(FIXTURE);+ return settled.length >= 2 && settled.every((turn) => turn.userText === TOOL_PROMPT);+ } catch {+ return false;+ }+ },+ {+ timeout: 15_000,+ intervals: [250, 500, 1_000],+ message: 'recording should settle with several invocations under one prompt',+ },+ )+ .toBe(true);+ const recorded = fixtureTurns(FIXTURE);+ expect(+ recorded.map((turn) => turn.userText),+ 'every invocation of this turn shares its one user prompt',+ ).toEqual(recorded.map(() => TOOL_PROMPT));+ expect(+ recorded[0].toolCallChunkCount,+ 'the first invocation should stream the tool call',+ ).toBeGreaterThan(0);+ expect(+ namesTool(recorded[0].toolNames),+ `the streamed tool call should name ${TOOL_NAME}, got ${JSON.stringify(recorded[0].toolNames)}`,+ ).toBe(true);+ expect(+ recorded[recorded.length - 1].contentChunkCount,+ 'the post-tool invocation should stream the answer',+ ).toBeGreaterThan(0);+ } else {+ const turns = fixtureTurns(FIXTURE);+ expect(+ turns.length,+ 'the fixture should hold more than one invocation for this single turn',+ ).toBeGreaterThan(1);+ expect(+ namesTool(turns[0].toolNames),+ `the recorded tool call should name ${TOOL_NAME}, got ${JSON.stringify(turns[0].toolNames)}`,+ ).toBe(true);++ /** Replay drives the real tool node, so the tool ran again in this run+ * rather than being replayed as recorded output. */+ const toolResult = JSON.stringify(assistant[0].content ?? []);+ expect(toolResult, 'the replayed tool call should carry its real result').toContain(FACT);++ const ledger = readReplayLedger(FIXTURE);+ expect(+ ledger.invocationsConsumed,+ 'replay should advance through every invocation of the turn',+ ).toBe(ledger.invocationsTotal);+ assertFixtureConsumed(FIXTURE);+ }++ expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0);+ });+});diff --git a/e2e/specs/mock/model-replay.spec.ts b/e2e/specs/mock/model-replay.spec.tsnew file mode 100644index 0000000000..03131cdfb9--- /dev/null+++ b/e2e/specs/mock/model-replay.spec.ts@@ -0,0 +1,191 @@+import { expect, test } from '@playwright/test';+import type { TMessage } from 'librechat-data-provider';+import {+ assertFixtureConsumed,+ fixtureTurns,+ readReplayLedger,+ removeFixture,+} from './replay.helpers';+import {+ NEW_CHAT_PATH,+ fetchJson,+ messagesView,+ getAccessToken,+ selectMockEndpoint,+ sendMessageAndWaitForCompletion,+} from './helpers';++/**+ * Record-once/replay-forever coverage: one recorded real-provider conversation+ * replays keylessly through the real createRun → registered replay provider →+ * graph → SSE → persistence chain.+ *+ * Record (writes the fixture; needs a real provider key):+ * E2E_MODEL_FIXTURES=record E2E_MODEL_FIXTURE_NAME=deepseek-two-turn \+ * E2E_RECORD_PROVIDER_API_KEY=<key> \+ * npx playwright test --config=e2e/playwright.config.mock.ts model-replay+ *+ * Replay (default, keyless): the same drive steps; the conversation binds to+ * the committed fixture by prompt text, assistant turns must equal the+ * recorded turns exactly, and the consumption ledger must drain completely.+ */+const COMMITTED_FIXTURE = 'deepseek-two-turn';+const RECORDING = process.env.E2E_MODEL_FIXTURES === 'record';+/**+ * Record mode collects every replay spec, so each one records only the fixture+ * it owns and stands down for the others. A spec must never write a fixture+ * other than its own: two fixtures carrying the same prompts would make the+ * server-side binding ambiguous and refuse both.+ */+const RECORDING_THIS_FIXTURE =+ RECORDING && process.env.E2E_MODEL_FIXTURE_NAME === COMMITTED_FIXTURE;+const FIXTURE = COMMITTED_FIXTURE;+const RECORD_ENDPOINT = {+ label: 'Replay Record Provider',+ model: process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat',+};++/**+ * The closing prompt deliberately asks for prose: a one-token answer streams+ * as a single content delta wrapped in empty initialization and usage frames,+ * which cannot demonstrate incremental content streaming however many chunks+ * the provider emits around it.+ */+const TURN_PROMPTS = [+ 'Name the two prime numbers between 20 and 30, comma separated, and nothing else.',+ 'In two short sentences, explain why the sum of those two primes is an even number. Begin with the word "Because".',+];++test.describe('recorded model fixture replay', () => {+ test('a recorded conversation replays deterministically through the real pipeline', async ({+ page,+ }) => {+ test.skip(+ RECORDING && !RECORDING_THIS_FIXTURE,+ `recording ${process.env.E2E_MODEL_FIXTURE_NAME}`,+ );+ test.setTimeout(180_000);+ const pageErrors: string[] = [];+ page.on('pageerror', (error) => pageErrors.push(error.message));++ await page.goto(NEW_CHAT_PATH, { timeout: 10_000 });+ if (RECORDING) {+ /** Proof of a fresh write: the assertions below cannot be satisfied by a+ * pre-existing fixture, so a run whose recorder never installed fails+ * instead of greening against a stale artifact. */+ removeFixture(FIXTURE);+ await selectMockEndpoint(page, RECORD_ENDPOINT);+ }++ for (const prompt of TURN_PROMPTS) {+ await sendMessageAndWaitForCompletion(page, prompt, { timeout: 90_000 });+ }++ const conversationId = /\/c\/([^/]+)/.exec(page.url())?.[1];+ expect(conversationId, 'conversation should have a persisted id').toBeTruthy();+ const token = await getAccessToken(page);+ const messages = await fetchJson<TMessage[]>(+ page,+ `/api/messages/${encodeURIComponent(conversationId as string)}`,+ token,+ );+ /** Agents-pipeline messages persist their text inside `content` parts;+ * top-level `text` stays empty. */+ const persistedText = (message: TMessage): string => {+ if (message.text) {+ return message.text;+ }+ return (message.content ?? [])+ .map((part) => {+ if (part?.type !== 'text') {+ return '';+ }+ const text = (part as { text?: string | { value?: string } }).text;+ return typeof text === 'string' ? text : (text?.value ?? '');+ })+ .join('');+ };+ const assistantTexts = messages+ .filter((message) => message.isCreatedByUser === false)+ .map(persistedText);+ expect(assistantTexts).toHaveLength(TURN_PROMPTS.length);++ if (RECORDING) {+ /** The recorder rides LangChain token callbacks, which the provider+ * stream dispatches without awaiting — the recording quiesces shortly+ * AFTER the durable-completion barrier, so the record-mode harvest+ * polls for the settled fixture instead of asserting a single read.+ * Replay mode needs no such poll: the replaying generator finishes its+ * ledger writes before the turn can persist. */+ await expect+ .poll(+ () => {+ try {+ const settled = fixtureTurns(FIXTURE);+ return (+ settled.length === TURN_PROMPTS.length &&+ settled.every(+ (turn, index) =>+ turn.userText === TURN_PROMPTS[index] &&+ turn.finalText === assistantTexts[index],+ )+ );+ } catch {+ return false;+ }+ },+ {+ timeout: 15_000,+ intervals: [250, 500, 1_000],+ message: 'recorded fixture should quiesce with the persisted assistant turns',+ },+ )+ .toBe(true);+ const recorded = fixtureTurns(FIXTURE);+ for (const turn of recorded) {+ expect(turn.contentChunkCount, 'each turn should record assistant content').toBeGreaterThan(+ 0,+ );+ }+ expect(+ recorded[recorded.length - 1].contentChunkCount,+ 'the prose turn should record several content deltas, not one delta padded with empty frames',+ ).toBeGreaterThan(1);+ } else {+ const turns = fixtureTurns(FIXTURE);+ expect(+ turns.map((turn) => turn.userText),+ 'fixture invocations should mirror the driven prompts',+ ).toEqual(TURN_PROMPTS);+ expect(assistantTexts, 'replayed assistant turns should equal the recording exactly').toEqual(+ turns.map((turn) => turn.finalText),+ );+ /** Compare against rendered markdown, not the raw recording: a reply+ * opening with `52.` is rendered as an ordered-list marker and never+ * appears in the DOM text, so a leading enumerator is stripped before+ * matching and only a prose prefix is used. */+ const renderedPrefix = turns[turns.length - 1].finalText+ .replace(/^\s*\d+[.)]\s*/, '')+ .trim()+ .slice(0, 30);+ expect(+ renderedPrefix.length,+ 'the prose turn should yield a comparable prefix',+ ).toBeGreaterThan(15);+ await expect(messagesView(page)).toContainText(renderedPrefix);++ /** Streaming incrementality is asserted from the ledger's drained chunk+ * count, not by sampling transient DOM — every recorded chunk passed+ * through the live SSE wire before the durable completion barrier. */+ const ledger = readReplayLedger(FIXTURE);+ expect(ledger.chunksConsumed).toBe(ledger.chunksTotal);+ expect(+ turns[turns.length - 1].contentChunkCount,+ 'replay should stream several content deltas for the prose turn',+ ).toBeGreaterThan(1);+ assertFixtureConsumed(FIXTURE);+ }++ expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0);+ });+});diff --git a/e2e/specs/mock/replay.helpers.ts b/e2e/specs/mock/replay.helpers.tsnew file mode 100644index 0000000000..042ded965b--- /dev/null+++ b/e2e/specs/mock/replay.helpers.ts@@ -0,0 +1,145 @@+import fs from 'fs';+import path from 'path';++/**+ * Spec-side readers for the model-fixture replay lane. The server-side+ * recorder/replayer (`e2e/setup/model-replay.js`) owns the formats; these+ * readers stay dependency-free on that CJS module so the spec plane needs no+ * runtime import of server code.+ */+const FIXTURES_DIR = path.resolve(__dirname, '../../fixtures/model-replay');+const LEDGER_DIR = path.resolve(__dirname, '../.test-results/model-replay');++export type FixtureTurn = {+ userText: string;+ finalText: string;+ chunkCount: number;+ /**+ * Chunks carrying assistant text, as distinct from the empty+ * initialization and usage-metadata chunks a provider also emits. Only+ * these prove incremental content streaming — a total chunk count above one+ * is satisfied by a single content delta wrapped in empty frames.+ */+ contentChunkCount: number;+ /** Chunks carrying `tool_call_chunks`, i.e. the streamed tool invocation. */+ toolCallChunkCount: number;+ /** Tool names streamed by this invocation, in order of first appearance. */+ toolNames: string[];+};++export type ReplayLedger = {+ fixture: string;+ invocationsTotal: number;+ chunksTotal: number;+ invocationsConsumed: number;+ chunksConsumed: number;+ overruns: Array<{ at: string; userText: string }>;+ promptMismatches: Array<{ invocation: number; expected: string; received: string }>;+};++export function fixturePath(name: string): string {+ return path.join(FIXTURES_DIR, `${name}.jsonl`);+}++/**+ * Remove a fixture before a recording run so its assertions cannot be+ * satisfied by a pre-existing artifact. Without this, a run whose hook failed+ * to install the recorder would still see the live provider answer these+ * deterministic prompts while the poll read the stale file — matching answers,+ * valid chunk counts, and a green run that wrote nothing.+ */+export function removeFixture(name: string): void {+ fs.rmSync(fixturePath(name), { force: true });+}++/**+ * Parse a fixture's invocations in recorded order. An invocation's final text+ * is the concatenation of its recorded chunk texts — the chunks are written+ * synchronously during the stream, while the provider's `handleLLMEnd`+ * dispatch (the `end` line) can land after the durable-completion barrier a+ * spec waits on, so nothing here depends on it.+ */+export function fixtureTurns(name: string): FixtureTurn[] {+ const lines = fs+ .readFileSync(fixturePath(name), 'utf8')+ .split('\n')+ .filter(Boolean)+ .map((line) => JSON.parse(line) as Record<string, unknown>);+ const turns: FixtureTurn[] = [];+ for (const entry of lines) {+ if (entry.type === 'invocation') {+ turns[entry.index as number] = {+ userText: entry.userText as string,+ finalText: '',+ chunkCount: 0,+ contentChunkCount: 0,+ toolCallChunkCount: 0,+ toolNames: [],+ };+ } else if (entry.type === 'chunk') {+ const turn = turns[entry.invocation as number];+ if (turn) {+ const text = (entry.text as string) ?? '';+ turn.chunkCount += 1;+ turn.finalText += text;+ if (text !== '') {+ turn.contentChunkCount += 1;+ }+ const message = entry.message as+ | { tool_call_chunks?: Array<{ name?: string }> }+ | undefined;+ const toolCallChunks = message?.tool_call_chunks ?? [];+ if (toolCallChunks.length > 0) {+ turn.toolCallChunkCount += 1;+ for (const call of toolCallChunks) {+ if (call.name && !turn.toolNames.includes(call.name)) {+ turn.toolNames.push(call.name);+ }+ }+ }+ }+ } else if (entry.type === 'error') {+ throw new Error(`Fixture ${name} recorded a provider error: ${String(entry.message)}`);+ }+ }+ return turns;+}++export function readReplayLedger(name: string): ReplayLedger {+ const ledgerPath = path.join(LEDGER_DIR, `${name}.json`);+ if (!fs.existsSync(ledgerPath)) {+ throw new Error(+ `Replay ledger missing for fixture "${name}" (${ledgerPath}); ` ++ 'the conversation never bound to the fixture',+ );+ }+ return JSON.parse(fs.readFileSync(ledgerPath, 'utf8')) as ReplayLedger;+}++/**+ * The teardown consumption check: every recorded invocation and chunk was+ * drained, nothing was invoked past the script, and every prompt matched its+ * recording. Converts silent underruns and shifted bindings into crisp+ * diagnostics.+ */+export function assertFixtureConsumed(name: string): void {+ const ledger = readReplayLedger(name);+ const failures: string[] = [];+ if (ledger.invocationsConsumed !== ledger.invocationsTotal) {+ failures.push(+ `under-consumed: ${ledger.invocationsConsumed}/${ledger.invocationsTotal} invocations`,+ );+ }+ if (ledger.chunksConsumed !== ledger.chunksTotal) {+ failures.push(`under-streamed: ${ledger.chunksConsumed}/${ledger.chunksTotal} chunks`);+ }+ if (ledger.overruns.length > 0) {+ failures.push(`over-consumed ${ledger.overruns.length}x: ${JSON.stringify(ledger.overruns)}`);+ }+ if (ledger.promptMismatches.length > 0) {+ failures.push(`prompt mismatches: ${JSON.stringify(ledger.promptMismatches)}`);+ }+ if (failures.length > 0) {+ throw new Error(`Fixture "${name}" consumption check failed — ${failures.join('; ')}`);+ }+}diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.tsindex 13b2725986..029807ebba 100644--- a/packages/api/src/agents/run.ts+++ b/packages/api/src/agents/run.ts@@ -1301,6 +1301,7 @@ function buildSubagentConfigs(export async function createRun({runId,signal,+ conversationId,agents,messages,discoveredToolNames,@@ -1331,6 +1332,9 @@ export async function createRun({agents: RunAgent[];signal: AbortSignal;runId?: string;+ /** Conversation-stable identity, used by the e2e run hook to tell a resumed+ * run apart from a fresh attempt (a resume carries no messages). */+ conversationId?: string;streaming?: boolean;streamUsage?: boolean;requestBody?: t.RequestBody;@@ -1991,6 +1995,11 @@ export async function createRun({const run = await Run.create(runConfig);applyCustomHandoffPromptKeyCompatibility(run, runConfig.graphConfig);- applyTestRunHook(run, { messages, agents, modelCallbacks });+ applyTestRunHook(run, {+ messages,+ agents,+ modelCallbacks,+ conversationId: conversationId ?? requestBody?.conversationId ?? undefined,+ });return run;}diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.tsindex ef0a83d968..487369aa86 100644--- a/packages/api/src/agents/testHook.ts+++ b/packages/api/src/agents/testHook.ts@@ -9,6 +9,13 @@ import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent*/export interface TestRunHookContext {messages?: BaseMessage[];+ /**+ * Identifies the conversation this run belongs to. A resumed run rebuilds+ * `createRun` with no messages because state is rehydrated from the+ * checkpoint, so message history alone cannot tell a resume apart from a+ * fresh attempt; this is the stable identity across both.+ */+ conversationId?: string;agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>;modelCallbacks?: readonly ModelBoundChatModelCallback[];}