Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🎬 test: Record-Once/Replay-Forever Model Fixtures for Mock E2E (#15210)

Danny Avila1489623fa3aa
diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js
index 6c1b6a6136..42195a0c3e 100644
--- a/api/server/controllers/agents/client.js
+++ b/api/server/controllers/agents/client.js
@@ -3600,6 +3600,9 @@ class AgentClient extends BaseClient {
(activityLabel ? createAssistantPhaseStampingHandlers(offsetHandlers) : offsetHandlers);
const createRunPromise = createRun({
agents,
+ // Conversation-stable identity for the e2e run hook; a resumed run
+ // carries no messages, so history cannot identify the conversation.
+ conversationId: this.conversationId,
messages,
modelCallbacks: [modelBoundCallback],
// This controller implements the full HITL pause/resume lifecycle (handleRunInterrupt
@@ -4070,6 +4073,7 @@ class AgentClient extends BaseClient {
(activityLabel ? createAssistantPhaseStampingHandlers(offsetHandlers) : offsetHandlers);
run = await createRun({
agents,
+ conversationId: this.conversationId,
modelCallbacks: [modelBoundCallback],
// State (messages, tool calls) is rehydrated from the checkpoint by
// run.resume; createRun only needs the agents to rebuild the graph.
diff --git a/e2e/config/librechat.e2e.yaml b/e2e/config/librechat.e2e.yaml
index 07dda8b569..92efa48463 100644
--- a/e2e/config/librechat.e2e.yaml
+++ b/e2e/config/librechat.e2e.yaml
@@ -92,6 +92,10 @@ endpoints:
- module: e2e/setup/tool-approval-hook.js
matcher: ^approval_probe_mcp_e2e-memory$
custom:
+ # Substituted with a REAL provider endpoint only when the mock config runs
+ # in model-fixture record mode (E2E_MODEL_FIXTURES=record); a comment
+ # otherwise, so the credential-free profile never gains a live endpoint.
+ # __E2E_MODEL_RECORD_PROVIDER__
- name: 'Mock Provider A'
apiKey: 'e2e-mock-key-a'
baseURL: 'http://127.0.0.1:8889/v1'
@@ -184,6 +188,7 @@ modelSpecs:
# Surfaces the endpoints menu (modelSelect defaults on when addedEndpoints is
# set) limited to entries that don't collide with the spec labels above.
addedEndpoints:
+ # __E2E_MODEL_RECORD_ADDED_ENDPOINT__
- 'Mock Provider C'
- 'Mock Provider D'
- 'Mock Provider E'
diff --git a/e2e/fixtures/model-replay/deepseek-tool-call.jsonl b/e2e/fixtures/model-replay/deepseek-tool-call.jsonl
new file mode 100644
index 0000000000..0ede2d819c
--- /dev/null
+++ b/e2e/fixtures/model-replay/deepseek-tool-call.jsonl
@@ -0,0 +1,40 @@
+{"type":"meta","name":"deepseek-tool-call","recordedAt":"2026-08-25T22:23:39.533Z"}
+{"type":"invocation","index":0,"userText":"Call the remember_fact tool with fact set to \"the replay lane records tool calls\", then reply with exactly the text the tool returned and nothing else."}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"id":"call_00_MvSzx7ML4G6RE6Twg4cr3997","type":"function","function":{"name":"remember_fact_mcp_e2e-memory","arguments":""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"name":"remember_fact_mcp_e2e-memory","args":"","id":"call_00_MvSzx7ML4G6RE6Twg4cr3997","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"{"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"{","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"fact"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"fact","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":": "}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":": ","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"the"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"the","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" replay"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" replay","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" lane"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" lane","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" records"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" records","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" tool"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" tool","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":" calls"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":" calls","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"\"","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[{"args":"}","index":0,"type":"tool_call_chunk"}],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"finish_reason":"tool_calls","system_fingerprint":"a26a7955944dc5c60445bff77fac9c8e","model_name":"deepseek-v4-flash","model_provider":"openai","usage":{"prompt_tokens":517,"completion_tokens":56,"total_tokens":573,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":5}},"tool_call_chunks":[],"id":"7adcadad-1d0a-42d1-8f62-ae2eea926eab"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":517,"completion_tokens":56,"total_tokens":573,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":5}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":517,"output_tokens":56,"total_tokens":573,"input_token_details":{"cache_read":512}},"id":"run-01a03b05-a093-744d-a6c6-dbf53d88db4a"}}
+{"type":"end","invocation":0,"text":""}
+{"type":"invocation","index":1,"userText":"Call the remember_fact tool with fact set to \"the replay lane records tool calls\", then reply with exactly the text the tool returned and nothing else."}
+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":"E","message":{"content":"E","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":"2","message":{"content":"2","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":"E","message":{"content":"E","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" M","message":{"content":" M","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":"CP","message":{"content":"CP","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" memory","message":{"content":" memory","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" noted","message":{"content":" noted","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":":","message":{"content":":","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" the","message":{"content":" the","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" replay","message":{"content":" replay","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" lane","message":{"content":" lane","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" records","message":{"content":" records","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" tool","message":{"content":" tool","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":" calls","message":{"content":" calls","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"prompt":0,"completion":0,"finish_reason":"stop","system_fingerprint":"a26a7955944dc5c60445bff77fac9c8e","model_name":"deepseek-v4-flash","model_provider":"openai","usage":{"prompt_tokens":599,"completion_tokens":14,"total_tokens":613,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":87}},"tool_call_chunks":[],"id":"e29c2dbf-a4a5-45a2-b868-01167671fc19"}}
+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":599,"completion_tokens":14,"total_tokens":613,"prompt_tokens_details":{"cached_tokens":512},"prompt_cache_hit_tokens":512,"prompt_cache_miss_tokens":87}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":599,"output_tokens":14,"total_tokens":613,"input_token_details":{"cache_read":512}},"id":"run-01a03b05-a445-728f-821b-b6b325e263e9"}}
+{"type":"end","invocation":1,"text":"E2E MCP memory noted: the replay lane records tool calls"}
diff --git a/e2e/fixtures/model-replay/deepseek-two-turn.jsonl b/e2e/fixtures/model-replay/deepseek-two-turn.jsonl
new file mode 100644
index 0000000000..056d14e4ac
--- /dev/null
+++ b/e2e/fixtures/model-replay/deepseek-two-turn.jsonl
@@ -0,0 +1,52 @@
+{"type":"meta","name":"deepseek-two-turn","recordedAt":"2026-08-25T17:44:11.310Z"}
+{"type":"invocation","index":0,"userText":"Name the two prime numbers between 20 and 30, comma separated, and nothing else."}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}
+{"type":"chunk","invocation":0,"text":"23","message":{"content":"23","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}
+{"type":"chunk","invocation":0,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}
+{"type":"chunk","invocation":0,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}
+{"type":"chunk","invocation":0,"text":"29","message":{"content":"29","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{"prompt_tokens":23,"completion_tokens":4,"total_tokens":27,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":23}},"tool_call_chunks":[],"id":"b9bb0d22-7426-4b11-9827-b5778609c5ec"}}
+{"type":"chunk","invocation":0,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":23,"completion_tokens":4,"total_tokens":27,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":23}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":23,"output_tokens":4,"total_tokens":27,"input_token_details":{"cache_read":0}}}}
+{"type":"end","invocation":0,"text":"23, 29"}
+{"type":"invocation","index":1,"userText":"In two short sentences, explain why the sum of those two primes is an even number. Begin with the word \"Because\"."}
+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"Because","message":{"content":"Because","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" any","message":{"content":" any","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" prime","message":{"content":" prime","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" number","message":{"content":" number","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" greater","message":{"content":" greater","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" than","message":{"content":" than","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"2","message":{"content":"2","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" odd","message":{"content":" odd","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" and","message":{"content":" and","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" the","message":{"content":" the","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" sum","message":{"content":" sum","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" of","message":{"content":" of","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" two","message":{"content":" two","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" odd","message":{"content":" odd","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" numbers","message":{"content":" numbers","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" always","message":{"content":" always","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" even","message":{"content":" even","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":".","message":{"content":".","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" Therefore","message":{"content":" Therefore","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"23","message":{"content":"23","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" plus","message":{"content":" plus","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"29","message":{"content":"29","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" equals","message":{"content":" equals","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" ","message":{"content":" ","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"52","message":{"content":"52","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":",","message":{"content":",","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" which","message":{"content":" which","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" is","message":{"content":" is","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":" even","message":{"content":" even","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":".","message":{"content":".","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"model_provider":"openai","usage":{"prompt_tokens":56,"completion_tokens":37,"total_tokens":93,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":56}},"tool_call_chunks":[],"id":"6d185292-b2ce-4e24-992f-8a6c1dc2cadf"}}
+{"type":"chunk","invocation":1,"text":"","message":{"content":"","additional_kwargs":{},"response_metadata":{"usage":{"prompt_tokens":56,"completion_tokens":37,"total_tokens":93,"prompt_tokens_details":{"cached_tokens":0},"prompt_cache_hit_tokens":0,"prompt_cache_miss_tokens":56}},"tool_call_chunks":[],"usage_metadata":{"input_tokens":56,"output_tokens":37,"total_tokens":93,"input_token_details":{"cache_read":0}}}}
+{"type":"end","invocation":1,"text":"Because any prime number greater than 2 is odd, and the sum of two odd numbers is always even. Therefore, 23 plus 29 equals 52, which is even."}
diff --git a/e2e/playwright.config.mock.ts b/e2e/playwright.config.mock.ts
index 694061c1b2..0787f72fc0 100644
--- a/e2e/playwright.config.mock.ts
+++ b/e2e/playwright.config.mock.ts
@@ -26,6 +26,67 @@ const labelServerPath = path.resolve(rootPath, 'e2e/setup/fake-label-server.js')
* `writeRuntimeMockConfig` substitutes any override into the generated copy. */
const LABEL_PORT = process.env.E2E_LABEL_PORT || '8889';
const fakeModelHookPath = path.resolve(rootPath, 'e2e/setup/fake-model.js');
+/** Model-fixture record mode: the run hook taps the REAL provider stream into a
+ * replayable fixture instead of overriding the model (e2e/setup/model-replay.js). */
+const modelFixtureRecording = process.env.E2E_MODEL_FIXTURES === 'record';
+const recordModelHookPath = path.resolve(rootPath, 'e2e/setup/record-model.js');
+const recordProviderBaseURL =
+ process.env.E2E_RECORD_PROVIDER_BASE_URL || 'https://api.deepseek.com/v1';
+const recordProviderModel = process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat';
+if (modelFixtureRecording && !process.env.E2E_RECORD_PROVIDER_API_KEY) {
+ throw new Error('E2E_MODEL_FIXTURES=record requires E2E_RECORD_PROVIDER_API_KEY');
+}
+/**
+ * Each fixture belongs to exactly one spec, and a spec records only its own.
+ * Accepting an arbitrary name would leave a second fixture beside the
+ * committed one carrying the same prompts, and the server-side ambiguity
+ * check would then refuse to bind either — a successful recording run would
+ * disable the keyless lane.
+ */
+const RECORDABLE_FIXTURES = ['deepseek-two-turn', 'deepseek-tool-call'];
+if (modelFixtureRecording && !process.env.E2E_MODEL_FIXTURE_NAME) {
+ throw new Error('E2E_MODEL_FIXTURES=record requires E2E_MODEL_FIXTURE_NAME');
+}
+if (
+ modelFixtureRecording &&
+ !RECORDABLE_FIXTURES.includes(process.env.E2E_MODEL_FIXTURE_NAME ?? '')
+) {
+ throw new Error(
+ `E2E_MODEL_FIXTURE_NAME must be one of ${RECORDABLE_FIXTURES.join(', ')}; ` +
+ `received ${process.env.E2E_MODEL_FIXTURE_NAME}`,
+ );
+}
+/**
+ * Playwright documents `-c` as an alias for `--config`, so both spellings are
+ * parsed — recognising only the long form would let the short one slip past.
+ *
+ * Derived configs (`playwright.config.redis.ts`, `.mermaid.ts`) spread this
+ * config and then replace `testMatch`, discarding the record-mode restriction
+ * below — their specs would reach the paid provider and rewrite the selected
+ * fixture. The restriction cannot be enforced through a value a consumer can
+ * overwrite, so record mode refuses any config but this one.
+ */
+if (modelFixtureRecording) {
+ /** Only the process that parsed the CLI carries `--config`; Playwright
+ * workers do not, and must not be judged on an argument they never saw. */
+ const configFlagIndex = process.argv.findIndex(
+ (arg) =>
+ arg === '--config' || arg === '-c' || arg.startsWith('--config=') || arg.startsWith('-c='),
+ );
+ const configFlag = configFlagIndex === -1 ? undefined : process.argv[configFlagIndex];
+ let configPath: string | undefined;
+ if (configFlag?.includes('=')) {
+ configPath = configFlag.slice(configFlag.indexOf('=') + 1);
+ } else if (configFlag) {
+ configPath = process.argv[configFlagIndex + 1];
+ }
+ if (configPath && !/playwright\.config\.mock\.ts$/.test(configPath)) {
+ throw new Error(
+ `E2E_MODEL_FIXTURES=record only runs under playwright.config.mock.ts, not ${configPath}; ` +
+ 'derived configs replace testMatch and would send their specs to the real provider',
+ );
+ }
+}
const assistantsServerPath = path.resolve(rootPath, 'e2e/setup/fake-assistants-server.js');
const ASSISTANTS_PORT = process.env.E2E_ASSISTANTS_PORT || '8890';
const configTemplatePath = path.resolve(rootPath, 'e2e/config/librechat.e2e.yaml');
@@ -61,8 +122,15 @@ const baseEnv = {
...getLocalE2EEnv(),
CONFIG_PATH: configPath,
DEPLOYMENT_SKILLS_DIR: deploymentSkillsPath,
- /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model. */
- LIBRECHAT_TEST_RUN_HOOK: fakeModelHookPath,
+ /** Loaded in-process by `@librechat/api`'s `createRun` to swap in a fake model —
+ * or, in model-fixture record mode, to tap the real provider stream. */
+ LIBRECHAT_TEST_RUN_HOOK: modelFixtureRecording ? recordModelHookPath : fakeModelHookPath,
+ ...(modelFixtureRecording
+ ? {
+ E2E_MODEL_FIXTURE_NAME: process.env.E2E_MODEL_FIXTURE_NAME ?? '',
+ E2E_RECORD_PROVIDER_API_KEY: process.env.E2E_RECORD_PROVIDER_API_KEY ?? '',
+ }
+ : {}),
...(enableDynamicMcp ? { E2E_MCP_LIST_CHANGED: 'true', E2E_MCP_STATE_PATH: MCP_STATE_PATH } : {}),
/** The Assistants runtime uses the OpenAI SDK directly, outside the agents run hook. */
ASSISTANTS_API_KEY: 'e2e-mock-assistants-key',
@@ -117,7 +185,27 @@ function writeRuntimeMockConfig() {
].join('\n'),
}
: { allowedDomain: '', stdioEnv: '', networkServers: '' };
+ const recordProviderBlock = modelFixtureRecording
+ ? [
+ `- name: 'Replay Record Provider'`,
+ ` apiKey: '\${E2E_RECORD_PROVIDER_API_KEY}'`,
+ ` baseURL: '${recordProviderBaseURL}'`,
+ ' models:',
+ ' default:',
+ ` - '${recordProviderModel}'`,
+ ' fetch: false',
+ ' titleConvo: false',
+ ` modelDisplayLabel: 'Replay Record Provider'`,
+ ].join('\n ')
+ : '# __E2E_MODEL_RECORD_PROVIDER__';
config = config
+ .replace('# __E2E_MODEL_RECORD_PROVIDER__', recordProviderBlock)
+ .replace(
+ '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__',
+ modelFixtureRecording
+ ? `- 'Replay Record Provider'`
+ : '# __E2E_MODEL_RECORD_ADDED_ENDPOINT__',
+ )
.replace('# __E2E_DYNAMIC_MCP_ALLOWED_DOMAIN__', dynamicMcpConfig.allowedDomain)
.replace('# __E2E_DYNAMIC_MCP_STDIO_ENV__', dynamicMcpConfig.stdioEnv)
.replace('# __E2E_DYNAMIC_MCP_NETWORK_SERVERS__', dynamicMcpConfig.networkServers);
@@ -177,6 +265,12 @@ export default defineConfig({
globalSetup: require.resolve('./setup/global-setup'),
globalTeardown: require.resolve('./setup/global-teardown.mock'),
testDir: 'specs/mock/',
+ /** Record mode swaps the fake model for a real provider, so it must never
+ * run the whole mock suite: every spec's prompts would reach the paid
+ * endpoint, and each fresh conversation would truncate and rewrite the one
+ * selected fixture, leaving whichever scenario ran last. Without this an
+ * unfiltered entry point (`npm run e2e:mock`) does exactly that. */
+ ...(modelFixtureRecording ? { testMatch: /model-replay[a-z-]*\.spec\.ts$/ } : {}),
outputDir: 'specs/.test-results',
fullyParallel: true,
forbidOnly: !!process.env.CI,
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js
index a6e69ac67d..26e31c63ec 100644
--- a/e2e/setup/fake-model.js
+++ b/e2e/setup/fake-model.js
@@ -12,6 +12,7 @@
const { FakeChatModel } = require('@librechat/agents');
const { ChatGenerationChunk } = require('@langchain/core/outputs');
const { AIMessageChunk } = require('@langchain/core/messages');
+const { tryBindReplay } = require('./model-replay');
const MOCK_REPLY = process.env.MOCK_LLM_REPLY || 'E2E mock reply: pong';
const CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;
@@ -2393,6 +2394,21 @@ module.exports = function fakeModelHook(run, context) {
}
const text = getLatestUserText(context?.messages);
+ /** Recorded-session replay outranks marker routing: a conversation whose
+ * prompt matches a fixture's next recorded invocation streams that recording
+ * through the real pipeline instead of a scripted mock response. */
+ if (
+ tryBindReplay({
+ graph,
+ text,
+ agents: context?.agents,
+ messages: context?.messages,
+ conversationId: context?.conversationId,
+ modelCallbacks: context?.modelCallbacks,
+ })
+ ) {
+ return;
+ }
const toolNames = collectToolNames(context?.agents);
const handoffScript = parseHandoffScript(text);
const {
diff --git a/e2e/setup/model-replay.js b/e2e/setup/model-replay.js
new file mode 100644
index 0000000000..50ca0eab4e
--- /dev/null
+++ b/e2e/setup/model-replay.js
@@ -0,0 +1,732 @@
+/**
+ * Record-once/replay-forever model fixtures for the mock e2e harness.
+ *
+ * Record mode (`E2E_MODEL_FIXTURES=record` + `E2E_MODEL_FIXTURE_NAME=<name>`):
+ * `record-model.js` replaces the fake-model run hook; instead of overriding the
+ * graph's model it appends a LangChain callback handler to every agent
+ * context's `clientOptions.callbacks`, so the REAL provider model carries the
+ * recorder. Each model invocation's streamed `ChatGenerationChunk`s are
+ * serialized to `e2e/fixtures/model-replay/<name>.jsonl` exactly as the
+ * provider emitted them (text deltas, tool_call_chunks, reasoning
+ * additional_kwargs, usage_metadata). Only the invocation's latest human text
+ * is recorded for binding — system prompts and tool schemas never enter the
+ * fixture.
+ *
+ * Replay mode (default, keyless): `fake-model.js` consults `tryBindReplay`
+ * before its marker routing. A conversation binds to a fixture when its latest
+ * user text equals the fixture's next unconsumed invocation's recorded user
+ * text. The replaying model is not hand-assigned: `ReplayChatModel` is
+ * registered as the SDK provider `librechat-e2e-replay` via the agents
+ * package's `registerProvider`, and the bound instance is constructed through
+ * the SDK's own `initializeModel` — registry lookup, constructor
+ * `clientOptions` (carrying the model-bound callbacks the way
+ * `withModelCallbacks` does), and real `bindTools` over the run's tools — so
+ * the recorded chunks stream through the same SDK machinery a live provider
+ * uses: createRun → registered provider model → graph → SSE → persistence.
+ * Every invocation re-checks its prompt against the recording, an invocation
+ * past the end of the script throws (over-consumption fails loud in the
+ * turn), and a per-fixture consumption ledger under
+ * `e2e/specs/.test-results/model-replay/` lets specs assert at teardown that
+ * every recorded invocation and chunk was drained (under-consumption fails the
+ * spec, not silently).
+ *
+ * Constraint carried over from the recording model: one live binding per
+ * fixture per server process — scenarios replaying the same fixture must not
+ * run concurrently.
+ */
+const fs = require('fs');
+const path = require('path');
+const { FakeChatModel, registerProvider, initializeModel } = require('@librechat/agents');
+const { ChatGenerationChunk } = require('@langchain/core/outputs');
+const { AIMessageChunk } = require('@langchain/core/messages');
+
+const FIXTURES_DIR = path.resolve(__dirname, '../fixtures/model-replay');
+const LEDGER_DIR = path.resolve(__dirname, '../specs/.test-results/model-replay');
+const RECORDER_HANDLER_NAME = 'librechat-e2e-model-recorder';
+const SUMMARIZATION_GUARD_NAME = 'librechat-e2e-summarization-guard';
+const REPLAY_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_CHUNK_DELAY_MS) || 10;
+
+function extractText(content) {
+ if (typeof content === 'string') {
+ return content;
+ }
+ if (!Array.isArray(content)) {
+ return '';
+ }
+ const parts = [];
+ for (const part of content) {
+ if (typeof part === 'string') {
+ parts.push(part);
+ } else if (part && typeof part.text === 'string') {
+ parts.push(part.text);
+ }
+ }
+ return parts.join('');
+}
+
+function messageType(message) {
+ if (typeof message?.getType === 'function') {
+ return message.getType();
+ }
+ if (typeof message?._getType === 'function') {
+ return message._getType();
+ }
+ return message?.role;
+}
+
+/** Every human message's text, oldest first. */
+function humanTexts(messages) {
+ if (!Array.isArray(messages)) {
+ return [];
+ }
+ const texts = [];
+ for (const message of messages) {
+ const type = messageType(message);
+ if (type === 'human' || type === 'user') {
+ texts.push(extractText(message.content));
+ }
+ }
+ return texts;
+}
+
+/** The latest human message's text — the binding and prompt-check key. */
+function latestHumanText(messages) {
+ const texts = humanTexts(messages);
+ return texts.length > 0 ? texts[texts.length - 1] : '';
+}
+
+/**
+ * The distinct user turns a fixture records, in order. A turn that calls a
+ * tool spans several model invocations under one prompt, so the invocation
+ * sequence is not the turn sequence and only this collapsed view can be
+ * compared against a conversation's human messages.
+ */
+function fixtureTurnTexts(invocations) {
+ const turns = [];
+ for (const invocation of invocations) {
+ if (turns[turns.length - 1] !== invocation.userText) {
+ turns.push(invocation.userText);
+ }
+ }
+ return turns;
+}
+
+/**
+ * Whether this conversation is the one that already drove the fixture: its
+ * human turns open with exactly the fixture's recorded turns, in order. A
+ * consumed binding has to be retained for such a conversation, or an extra
+ * user turn would find no next invocation, fall through to ordinary
+ * fake-model routing, and be answered with a mock reply — leaving the
+ * over-consumption guard unreached and the drained ledger still passing.
+ */
+function conversationDroveFixture(messages, fixture) {
+ const texts = humanTexts(messages);
+ const turns = fixture.turns;
+ if (texts.length <= turns.length) {
+ return false;
+ }
+ return turns.every((turn, index) => turn === texts[index]);
+}
+
+function jsonClone(value) {
+ if (value == null) {
+ return undefined;
+ }
+ try {
+ return JSON.parse(JSON.stringify(value));
+ } catch {
+ return undefined;
+ }
+}
+
+/** Minimal AIMessageChunk projection that reconstructs the streamed message. */
+function serializeChunk(chunk, token) {
+ const message = chunk?.message;
+ const serialized = { text: chunk?.text ?? token ?? '' };
+ if (message) {
+ serialized.message = {
+ content: jsonClone(message.content) ?? '',
+ additional_kwargs: jsonClone(message.additional_kwargs),
+ response_metadata: jsonClone(message.response_metadata),
+ tool_call_chunks: jsonClone(message.tool_call_chunks),
+ usage_metadata: jsonClone(message.usage_metadata),
+ id: typeof message.id === 'string' ? message.id : undefined,
+ };
+ }
+ return serialized;
+}
+
+function deserializeChunk(serialized) {
+ const recorded = serialized.message;
+ const message = new AIMessageChunk({
+ content: recorded?.content ?? serialized.text ?? '',
+ additional_kwargs: recorded?.additional_kwargs ?? {},
+ response_metadata: recorded?.response_metadata ?? {},
+ tool_call_chunks: recorded?.tool_call_chunks ?? [],
+ usage_metadata: recorded?.usage_metadata,
+ id: recorded?.id,
+ });
+ return new ChatGenerationChunk({ text: serialized.text ?? '', message });
+}
+
+/* ------------------------------- recording ------------------------------- */
+
+const recordingState = {
+ initialized: false,
+ fixturePath: undefined,
+ invocationCounter: 0,
+ conversationId: undefined,
+ /** Bumped on every (re)start so handlers left on a superseded graph can be
+ * told apart from the current attempt's. */
+ generation: 0,
+ runIdToInvocation: new Map(),
+};
+
+function appendFixtureLine(entry) {
+ fs.appendFileSync(recordingState.fixturePath, `${JSON.stringify(entry)}\n`);
+}
+
+function initializeRecording(fixtureName) {
+ fs.mkdirSync(FIXTURES_DIR, { recursive: true });
+ recordingState.fixturePath = path.join(FIXTURES_DIR, `${fixtureName}.jsonl`);
+ fs.writeFileSync(recordingState.fixturePath, '');
+ appendFixtureLine({
+ type: 'meta',
+ name: fixtureName,
+ recordedAt: new Date().toISOString(),
+ });
+ recordingState.initialized = true;
+ recordingState.invocationCounter = 0;
+ recordingState.conversationId = undefined;
+ recordingState.generation += 1;
+ recordingState.runIdToInvocation.clear();
+ console.log(`[e2e model-replay] recording fixture ${recordingState.fixturePath}`);
+}
+
+/**
+ * The recorder's state is process-global and the web server outlives a
+ * Playwright retry, so a failed attempt that already recorded invocations
+ * would otherwise leave the counter advanced: the retry appends 2/3 after
+ * 0/1 (or keeps a previous attempt's `error` line) and the fixture is
+ * unusable for replay.
+ *
+ * A new attempt is a new conversation. Identity comes from `conversationId`
+ * rather than from the prompt or the history: a turn that calls a tool
+ * invokes the model again under the same latest human message, and a resumed
+ * run after a tool-approval pause rebuilds `createRun` with no messages at
+ * all because state is rehydrated from the checkpoint. Both would look like
+ * fresh attempts to any text- or history-based rule, and truncate the fixture
+ * mid-turn.
+ */
+function isConversationStart(messages) {
+ return humanTexts(messages).length <= 1;
+}
+
+function startsNewRecording(conversationId, messages) {
+ if (recordingState.invocationCounter === 0) {
+ return false;
+ }
+ if (conversationId != null && recordingState.conversationId != null) {
+ return recordingState.conversationId !== conversationId;
+ }
+ return isConversationStart(messages);
+}
+
+/**
+ * Handlers are stamped with the recording generation they were installed for.
+ * A failed attempt can still have a provider call in flight when the retry
+ * resets the recording, and its graph keeps this handler: without the stamp
+ * that stale call would allocate an invocation index from the new attempt's
+ * counter, or append an `error` entry whose mapping was cleared, corrupting
+ * the freshly reset fixture.
+ */
+function createRecorderHandler() {
+ const generation = recordingState.generation;
+ const superseded = () => generation !== recordingState.generation;
+ return {
+ name: RECORDER_HANDLER_NAME,
+ generation,
+ /** Callbacks must settle before the model call resolves, or the `end`
+ * line races the durable-completion barrier the recording spec waits on
+ * (the same contract ModelBoundChatModelCallback declares). */
+ awaitHandlers: true,
+ raiseError: true,
+ handleChatModelStart(_llm, messageBatches, runId) {
+ if (superseded()) {
+ return;
+ }
+ const index = recordingState.invocationCounter++;
+ recordingState.runIdToInvocation.set(runId, index);
+ appendFixtureLine({
+ type: 'invocation',
+ index,
+ userText: latestHumanText(messageBatches?.[0]),
+ });
+ },
+ handleLLMNewToken(token, _idx, runId, _parentRunId, _tags, fields) {
+ const invocation = recordingState.runIdToInvocation.get(runId);
+ if (superseded() || invocation == null) {
+ return;
+ }
+ appendFixtureLine({
+ type: 'chunk',
+ invocation,
+ ...serializeChunk(fields?.chunk, token),
+ });
+ },
+ handleLLMEnd(output, runId) {
+ const invocation = recordingState.runIdToInvocation.get(runId);
+ if (superseded() || invocation == null) {
+ return;
+ }
+ recordingState.runIdToInvocation.delete(runId);
+ const generation = output?.generations?.[0]?.[0];
+ appendFixtureLine({
+ type: 'end',
+ invocation,
+ text: generation?.text ?? extractText(generation?.message?.content),
+ });
+ },
+ handleLLMError(error, runId) {
+ const invocation = recordingState.runIdToInvocation.get(runId);
+ recordingState.runIdToInvocation.delete(runId);
+ if (superseded()) {
+ return;
+ }
+ appendFixtureLine({
+ type: 'error',
+ invocation: invocation ?? null,
+ message: error instanceof Error ? error.message : String(error),
+ });
+ },
+ };
+}
+
+/**
+ * Attach the recorder to every agent context's model client options. The model
+ * is created per-invocation from `agentContext.clientOptions`, so appending a
+ * callback here puts the recorder on the real provider stream without
+ * replacing the model.
+ */
+function installRecorder({ graph, messages, conversationId }) {
+ const fixtureName = process.env.E2E_MODEL_FIXTURE_NAME;
+ if (!fixtureName) {
+ console.warn('[e2e model-replay] E2E_MODEL_FIXTURE_NAME unset; not recording');
+ return;
+ }
+ if (!recordingState.initialized || startsNewRecording(conversationId, messages)) {
+ initializeRecording(fixtureName);
+ }
+ if (conversationId != null) {
+ recordingState.conversationId = conversationId;
+ }
+ const contexts = graph?.agentContexts;
+ if (!contexts || typeof contexts.values !== 'function') {
+ console.warn('[e2e model-replay] graph.agentContexts unavailable; not recording');
+ return;
+ }
+ for (const context of contexts.values()) {
+ if (!context.clientOptions) {
+ context.clientOptions = {};
+ }
+ attachRecorder(context.clientOptions);
+ /** Summarization runs on its own model with its own callback list.
+ * Recording those invocations without replaying them is worse than
+ * ignoring them: they would take slots in the fixture sequence that
+ * replay never consumes, so the next primary call would read the
+ * summariser's chunks. Replay routes only the agent model
+ * (`graph.overrideModel`) and subagents, so the honest boundary is to
+ * refuse a recording the lane could not reproduce. */
+ const summarizationParameters =
+ context.summarizationConfig?.parameters ?? context.summarizationConfig?.config?.parameters;
+ if (summarizationParameters) {
+ attachSummarizationGuard(summarizationParameters);
+ }
+ }
+}
+
+/**
+ * Fails a recording the moment the summarization model runs. Its invocations
+ * would otherwise enter the fixture sequence unreplayable — see
+ * `installRecorder`. Summarization fixtures need replay routing for that model
+ * before they can be supported.
+ */
+function attachSummarizationGuard(options) {
+ const handler = {
+ name: SUMMARIZATION_GUARD_NAME,
+ raiseError: true,
+ awaitHandlers: true,
+ handleChatModelStart() {
+ throw new Error(
+ '[e2e model-replay] summarization ran during recording, and replay cannot route the ' +
+ 'summarization model — its invocations would desynchronise the fixture. Record a ' +
+ 'scenario that stays under the context-pruning threshold.',
+ );
+ },
+ };
+ const existing = options.callbacks;
+ if (Array.isArray(existing)) {
+ if (!existing.some((entry) => entry?.name === SUMMARIZATION_GUARD_NAME)) {
+ options.callbacks = [...existing, handler];
+ }
+ return;
+ }
+ if (existing == null) {
+ options.callbacks = [handler];
+ return;
+ }
+ if (
+ typeof existing.addHandler === 'function' &&
+ !existing.handlers?.some((entry) => entry?.name === SUMMARIZATION_GUARD_NAME)
+ ) {
+ existing.addHandler(handler);
+ }
+}
+
+/** Append the recorder to a client-options object's callbacks, once. */
+function attachRecorder(options) {
+ /** Dedupe against the CURRENT generation only: a graph carried across a
+ * recording restart still holds a superseded handler, which is inert, so
+ * matching on name alone would leave that options object recording
+ * nothing. */
+ const isCurrent = (handler) =>
+ handler?.name === RECORDER_HANDLER_NAME && handler.generation === recordingState.generation;
+ const existing = options.callbacks;
+ if (Array.isArray(existing)) {
+ if (!existing.some(isCurrent)) {
+ options.callbacks = [
+ ...existing.filter((handler) => handler?.name !== RECORDER_HANDLER_NAME),
+ createRecorderHandler(),
+ ];
+ }
+ return;
+ }
+ if (existing == null) {
+ options.callbacks = [createRecorderHandler()];
+ return;
+ }
+ if (typeof existing.addHandler === 'function') {
+ if (!existing.handlers?.some(isCurrent)) {
+ existing.addHandler(createRecorderHandler());
+ }
+ }
+}
+
+/* -------------------------------- replay --------------------------------- */
+
+/** name -> { meta, invocations: [{ userText, chunks: [], finalText }] } */
+let fixtureRegistry;
+/** name -> { cursor, chunksConsumed, ledger } */
+const replayState = new Map();
+
+function parseFixtureFile(filePath) {
+ const name = path.basename(filePath, '.jsonl');
+ const invocations = [];
+ let meta = { name };
+ const lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
+ for (const line of lines) {
+ const entry = JSON.parse(line);
+ if (entry.type === 'meta') {
+ meta = entry;
+ } else if (entry.type === 'invocation') {
+ invocations[entry.index] = { userText: entry.userText, chunks: [], finalText: '' };
+ } else if (entry.type === 'chunk') {
+ invocations[entry.invocation]?.chunks.push(entry);
+ } else if (entry.type === 'end') {
+ const invocation = invocations[entry.invocation];
+ if (invocation) {
+ invocation.finalText = entry.text ?? '';
+ }
+ } else if (entry.type === 'error') {
+ throw new Error(
+ `[e2e model-replay] fixture ${name} recorded a provider error (${entry.message}); ` +
+ 're-record it before replaying',
+ );
+ }
+ }
+ const missing = invocations.findIndex((invocation) => invocation == null);
+ if (missing !== -1) {
+ throw new Error(`[e2e model-replay] fixture ${name} is missing invocation ${missing}`);
+ }
+ /** The file name is the fixture's identity — it is what `E2E_MODEL_FIXTURE_NAME`
+ * selects, what the spec names, and what the ledger is written under. A
+ * recorded `meta.name` is descriptive only: trusting it would let a renamed
+ * or copied fixture collapse onto another's registry key and ledger. */
+ return { meta: { ...meta, name }, invocations, turns: fixtureTurnTexts(invocations) };
+}
+
+function loadFixtureRegistry() {
+ if (fixtureRegistry) {
+ return fixtureRegistry;
+ }
+ fixtureRegistry = new Map();
+ if (!fs.existsSync(FIXTURES_DIR)) {
+ return fixtureRegistry;
+ }
+ for (const file of fs.readdirSync(FIXTURES_DIR)) {
+ if (file.endsWith('.jsonl')) {
+ const fixture = parseFixtureFile(path.join(FIXTURES_DIR, file));
+ fixtureRegistry.set(fixture.meta.name, fixture);
+ }
+ }
+ return fixtureRegistry;
+}
+
+function writeLedger(name) {
+ const state = replayState.get(name);
+ if (!state) {
+ return;
+ }
+ fs.mkdirSync(LEDGER_DIR, { recursive: true });
+ fs.writeFileSync(
+ path.join(LEDGER_DIR, `${name}.json`),
+ `${JSON.stringify({ fixture: name, ...state.ledger }, null, 2)}\n`,
+ );
+}
+
+function freshReplayState(fixture) {
+ return {
+ cursor: 0,
+ ledger: {
+ invocationsTotal: fixture.invocations.length,
+ chunksTotal: fixture.invocations.reduce(
+ (total, invocation) => total + invocation.chunks.length,
+ 0,
+ ),
+ invocationsConsumed: 0,
+ chunksConsumed: 0,
+ overruns: [],
+ promptMismatches: [],
+ },
+ };
+}
+
+function getReplayState(fixture) {
+ let state = replayState.get(fixture.meta.name);
+ if (!state) {
+ state = freshReplayState(fixture);
+ replayState.set(fixture.meta.name, state);
+ }
+ return state;
+}
+
+/**
+ * Start the fixture over for a new conversation. The web server outlives a
+ * Playwright retry, so without this a consumed cursor would leave the retry
+ * unable to bind its first prompt — it would fall through to marker routing
+ * and fail deterministically, burning every configured retry. The ledger
+ * resets with the cursor so the new attempt is judged on its own consumption
+ * rather than accumulating the previous one's counts.
+ */
+function restartReplayState(fixture) {
+ const state = freshReplayState(fixture);
+ replayState.set(fixture.meta.name, state);
+ return state;
+}
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const REPLAY_PROVIDER = 'librechat-e2e-replay';
+
+/**
+ * Constructed by the SDK's `initializeModel` through the provider registry, so
+ * `clientOptions` is the full constructor contract: the fixture binding, the
+ * shared cursor state, and the run's model-bound callbacks.
+ */
+class ReplayChatModel extends FakeChatModel {
+ constructor(clientOptions = {}) {
+ super({ responses: [''], sleep: 0, emitCustomEvent: false });
+ this.fixture = clientOptions.fixture;
+ this.state = clientOptions.state;
+ this.boundToolNames = clientOptions.boundToolNames ?? [];
+ if (clientOptions.callbacks) {
+ this.callbacks = clientOptions.callbacks;
+ }
+ }
+
+ /** Real SDK tool binding: returns a bound copy sharing the replay cursor. */
+ bindTools(tools) {
+ return new ReplayChatModel({
+ fixture: this.fixture,
+ state: this.state,
+ callbacks: this.callbacks,
+ boundToolNames: (tools ?? []).map((tool) => tool?.name ?? tool?.function?.name ?? 'unknown'),
+ });
+ }
+
+ async *_streamResponseChunks(messages, _options, runManager) {
+ const { fixture, state } = this;
+ const invocation = fixture.invocations[state.cursor];
+ if (!invocation) {
+ state.ledger.overruns.push({
+ at: new Date().toISOString(),
+ userText: latestHumanText(messages),
+ });
+ writeLedger(fixture.meta.name);
+ throw new Error(
+ `[e2e model-replay] fixture ${fixture.meta.name} over-consumed: model invoked ` +
+ `after all ${fixture.invocations.length} recorded invocations were drained`,
+ );
+ }
+ /** A resumed run carries no human message — state is rehydrated from the
+ * checkpoint — so there is no prompt to check against. Ownership already
+ * established which conversation this is; enforcing the recorded prompt
+ * here would reject every resume. Every real turn still gets checked. */
+ const promptText = latestHumanText(messages);
+ const carriesHumanTurn = humanTexts(messages).length > 0;
+ if (carriesHumanTurn && promptText !== invocation.userText) {
+ state.ledger.promptMismatches.push({
+ invocation: state.cursor,
+ expected: invocation.userText,
+ received: promptText,
+ });
+ writeLedger(fixture.meta.name);
+ throw new Error(
+ `[e2e model-replay] fixture ${fixture.meta.name} invocation ${state.cursor} ` +
+ `prompt mismatch: recorded ${JSON.stringify(invocation.userText)}, ` +
+ `received ${JSON.stringify(promptText)}`,
+ );
+ }
+ state.cursor += 1;
+ for (const chunk of invocation.chunks) {
+ await sleep(REPLAY_CHUNK_DELAY_MS);
+ yield deserializeChunk(chunk);
+ void runManager?.handleLLMNewToken(chunk.text ?? '');
+ state.ledger.chunksConsumed += 1;
+ }
+ state.ledger.invocationsConsumed += 1;
+ writeLedger(fixture.meta.name);
+ }
+}
+
+let replayProviderRegistered = false;
+
+function ensureReplayProviderRegistered() {
+ if (replayProviderRegistered) {
+ return;
+ }
+ try {
+ registerProvider({ provider: REPLAY_PROVIDER, model: ReplayChatModel });
+ } catch (error) {
+ /** The SDK registry is globalThis-scoped while this guard is
+ * module-scoped: a reloaded copy of this module finds the provider
+ * already registered. The registered class is stateless (fixture and
+ * cursor ride `clientOptions`), so any copy's registration serves all. */
+ if (!String(error instanceof Error ? error.message : error).includes('already registered')) {
+ throw error;
+ }
+ }
+ replayProviderRegistered = true;
+}
+
+/**
+ * Bind a conversation to a recorded fixture when its latest user text matches
+ * the fixture's next unconsumed invocation. The replay model is built through
+ * the SDK's registered-provider path (`registerProvider` +
+ * `initializeModel`), including real `bindTools` over the run's tools.
+ * Returns true when the graph's model was overridden with the replaying
+ * model; false lets the fake-model marker routing proceed unchanged.
+ */
+/**
+ * Decide how this run relates to a fixture.
+ *
+ * `own` — the conversation that claimed the fixture is back. Its cursor is
+ * authoritative wherever it stands, including past the end, so an extra turn
+ * reaches the over-consumption guard instead of falling through to the
+ * scripted fake model, and a resumed run after a tool-approval pause keeps
+ * replaying even though it arrives with no messages and no prompt text.
+ *
+ * `claim` — a different (or first) conversation opening the fixture: rewind
+ * and take ownership. This is what a Playwright retry looks like.
+ *
+ * Anything else is refused, so an unrelated conversation can never continue
+ * someone else's partly consumed script by happening to repeat a later prompt.
+ */
+function classifyBinding({ fixture, state, text, messages, conversationId }) {
+ if (conversationId != null && state.conversationId != null) {
+ if (state.conversationId === conversationId) {
+ return 'own';
+ }
+ return fixture.turns[0] === text ? 'claim' : 'refuse';
+ }
+ /** Identity unavailable (an older `@librechat/api` does not supply it):
+ * fall back to the text and history rules this lane used before. */
+ if (state.cursor !== 0 && isConversationStart(messages)) {
+ return fixture.turns[0] === text ? 'claim' : 'refuse';
+ }
+ if (fixture.invocations[state.cursor]?.userText === text) {
+ return 'own';
+ }
+ if (state.cursor !== 0 && fixture.turns[0] === text) {
+ return 'claim';
+ }
+ if (state.cursor >= fixture.invocations.length && conversationDroveFixture(messages, fixture)) {
+ return 'own';
+ }
+ return 'refuse';
+}
+
+function tryBindReplay({ graph, agents, text, messages, conversationId, modelCallbacks }) {
+ const registry = loadFixtureRegistry();
+ const matches = [];
+ for (const fixture of registry.values()) {
+ let state = getReplayState(fixture);
+ const binding = classifyBinding({ fixture, state, text, messages, conversationId });
+ if (binding === 'refuse') {
+ continue;
+ }
+ if (binding === 'claim') {
+ state = restartReplayState(fixture);
+ }
+ if (conversationId != null) {
+ state.conversationId = conversationId;
+ }
+ matches.push({ fixture, state });
+ }
+
+ if (matches.length === 0) {
+ return false;
+ }
+ /** Binding order would otherwise follow filesystem enumeration, so a second
+ * fixture sharing this prompt could silently redirect a scenario to the
+ * wrong chunks and ledger. The spec's fixture choice never reaches this
+ * server-side loop, so ambiguity has to fail rather than pick a winner. */
+ if (matches.length > 1) {
+ throw new Error(
+ `[e2e model-replay] prompt matches ${matches.length} fixtures ` +
+ `(${matches.map(({ fixture }) => fixture.meta.name).join(', ')}); ` +
+ 'fixtures must not share a bindable prompt',
+ );
+ }
+
+ const { fixture, state } = matches[0];
+ ensureReplayProviderRegistered();
+ const model = initializeModel({
+ provider: REPLAY_PROVIDER,
+ clientOptions: { fixture, state, callbacks: modelCallbacks },
+ tools: agents?.[0]?.tools ?? [],
+ });
+ state.ledger.toolsBound = model.boundToolNames ?? [];
+ graph.overrideModel = model;
+ /** `graph.overrideModel` is not inherited by child executors, so a fixture
+ * recording a subagent call — record mode captures child invocations, since
+ * the recorder attaches to every agent context — would otherwise leave the
+ * child on its configured provider: an underrun here, and a real provider
+ * request in a lane that must stay keyless. */
+ if (typeof graph.setSubagentModelOverride === 'function') {
+ graph.setSubagentModelOverride(model);
+ }
+ writeLedger(fixture.meta.name);
+ return true;
+}
+
+module.exports = {
+ FIXTURES_DIR,
+ LEDGER_DIR,
+ installRecorder,
+ tryBindReplay,
+ latestHumanText,
+ serializeChunk,
+ deserializeChunk,
+ parseFixtureFile,
+};
diff --git a/e2e/setup/record-model.js b/e2e/setup/record-model.js
new file mode 100644
index 0000000000..63ff76ab1d
--- /dev/null
+++ b/e2e/setup/record-model.js
@@ -0,0 +1,23 @@
+/**
+ * Run hook for `E2E_MODEL_FIXTURES=record`: taps the REAL provider model's
+ * stream instead of overriding it, writing each model invocation's chunks to
+ * `e2e/fixtures/model-replay/$E2E_MODEL_FIXTURE_NAME.jsonl` for keyless replay
+ * through `fake-model.js`. Set as `LIBRECHAT_TEST_RUN_HOOK` by the mock
+ * Playwright config when recording; the server must be booted with a working
+ * provider credential (`E2E_RECORD_PROVIDER_API_KEY`).
+ */
+const { installRecorder } = require('./model-replay');
+
+/** @type {import('@librechat/api').TestRunHook} */
+module.exports = function recordModelHook(run, context) {
+ const graph = run?.Graph;
+ if (!graph) {
+ console.warn('[e2e model-replay] record hook: run.Graph unavailable');
+ return;
+ }
+ installRecorder({
+ graph,
+ messages: context?.messages,
+ conversationId: context?.conversationId,
+ });
+};
diff --git a/e2e/specs/mock/model-replay-tools.spec.ts b/e2e/specs/mock/model-replay-tools.spec.ts
new file mode 100644
index 0000000000..df7d1f1c50
--- /dev/null
+++ b/e2e/specs/mock/model-replay-tools.spec.ts
@@ -0,0 +1,166 @@
+import { expect, test } from '@playwright/test';
+import type { Page } from '@playwright/test';
+import type { TMessage } from 'librechat-data-provider';
+import {
+ assertFixtureConsumed,
+ fixtureTurns,
+ readReplayLedger,
+ removeFixture,
+} from './replay.helpers';
+import {
+ NEW_CHAT_PATH,
+ fetchJson,
+ getAccessToken,
+ selectMockEndpoint,
+ sendMessageAndWaitForCompletion,
+} from './helpers';
+
+/**
+ * The tool-call half of the replay lane: one recorded turn in which the real
+ * provider calls an MCP tool, the tool runs for real, and the model is invoked
+ * a second time with its result.
+ *
+ * This is the shape a single prompt cannot express — one user turn spanning
+ * several model invocations — so it is what proves the fixture format carries
+ * `tool_call_chunks` and that replay advances through a turn's invocations
+ * rather than binding once per prompt.
+ *
+ * Record (needs a real provider key):
+ * E2E_MODEL_FIXTURES=record E2E_MODEL_FIXTURE_NAME=deepseek-tool-call \
+ * E2E_RECORD_PROVIDER_API_KEY=<key> \
+ * npx playwright test --config=e2e/playwright.config.mock.ts model-replay-tools
+ *
+ * Replay (default, keyless): the same drive steps against the committed
+ * fixture, with the recorded tool call streamed back through the real graph so
+ * the tool executes again.
+ */
+const COMMITTED_FIXTURE = 'deepseek-tool-call';
+const RECORDING = process.env.E2E_MODEL_FIXTURES === 'record';
+const RECORDING_THIS_FIXTURE =
+ RECORDING && process.env.E2E_MODEL_FIXTURE_NAME === COMMITTED_FIXTURE;
+const FIXTURE = COMMITTED_FIXTURE;
+const RECORD_ENDPOINT = {
+ label: 'Replay Record Provider',
+ model: process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat',
+};
+
+const MCP_SERVER_TITLE = 'E2E Memory';
+const TOOL_NAME = 'remember_fact';
+/** MCP tools reach the model under a server-qualified name
+ * (`remember_fact_mcp_e2e-memory`), and that qualification has changed before,
+ * so assertions match the base name as a prefix rather than pinning the suffix. */
+const namesTool = (names: string[]) => names.some((name) => name.startsWith(TOOL_NAME));
+/** The MCP fixture echoes this back, so the tool result is deterministic. */
+const FACT = 'the replay lane records tool calls';
+const TOOL_PROMPT =
+ `Call the ${TOOL_NAME} tool with fact set to "${FACT}", then reply with exactly the ` +
+ 'text the tool returned and nothing else.';
+
+/** Enable the ephemeral MCP server whose tools this turn calls. */
+async function selectEphemeralMCP(page: Page) {
+ await page.getByRole('button', { name: 'MCP Servers', exact: true }).click();
+ const serverItem = page.getByRole('menuitemcheckbox', { name: new RegExp(MCP_SERVER_TITLE) });
+ await expect(serverItem).toBeVisible();
+ await serverItem.click();
+ await expect(serverItem).toHaveAttribute('aria-checked', 'true');
+ await page.keyboard.press('Escape');
+ await expect(page.getByRole('button', { name: new RegExp(MCP_SERVER_TITLE) })).toBeVisible();
+}
+
+test.describe('recorded tool-call fixture replay', () => {
+ test('a recorded tool call replays through the real tool node', async ({ page }) => {
+ test.skip(
+ RECORDING && !RECORDING_THIS_FIXTURE,
+ `recording ${process.env.E2E_MODEL_FIXTURE_NAME}`,
+ );
+ test.setTimeout(180_000);
+ const pageErrors: string[] = [];
+ page.on('pageerror', (error) => pageErrors.push(error.message));
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10_000 });
+ if (RECORDING) {
+ removeFixture(FIXTURE);
+ await selectMockEndpoint(page, RECORD_ENDPOINT);
+ }
+ await selectEphemeralMCP(page);
+
+ await sendMessageAndWaitForCompletion(page, TOOL_PROMPT, { timeout: 120_000 });
+
+ const conversationId = /\/c\/([^/]+)/.exec(page.url())?.[1];
+ expect(conversationId, 'conversation should have a persisted id').toBeTruthy();
+ const token = await getAccessToken(page);
+ const messages = await fetchJson<TMessage[]>(
+ page,
+ `/api/messages/${encodeURIComponent(conversationId as string)}`,
+ token,
+ );
+ const assistant = messages.filter((message) => message.isCreatedByUser === false);
+ expect(assistant).toHaveLength(1);
+ /** The turn's durable proof that the tool ran: a persisted tool_call part
+ * naming the tool, independent of whatever prose the model wrapped it in. */
+ const toolCallParts = (assistant[0].content ?? []).filter((part) => part?.type === 'tool_call');
+ expect(toolCallParts.length, 'the turn should persist a tool call').toBeGreaterThan(0);
+ expect(JSON.stringify(toolCallParts)).toContain(TOOL_NAME);
+
+ if (RECORDING) {
+ await expect
+ .poll(
+ () => {
+ try {
+ const settled = fixtureTurns(FIXTURE);
+ return settled.length >= 2 && settled.every((turn) => turn.userText === TOOL_PROMPT);
+ } catch {
+ return false;
+ }
+ },
+ {
+ timeout: 15_000,
+ intervals: [250, 500, 1_000],
+ message: 'recording should settle with several invocations under one prompt',
+ },
+ )
+ .toBe(true);
+ const recorded = fixtureTurns(FIXTURE);
+ expect(
+ recorded.map((turn) => turn.userText),
+ 'every invocation of this turn shares its one user prompt',
+ ).toEqual(recorded.map(() => TOOL_PROMPT));
+ expect(
+ recorded[0].toolCallChunkCount,
+ 'the first invocation should stream the tool call',
+ ).toBeGreaterThan(0);
+ expect(
+ namesTool(recorded[0].toolNames),
+ `the streamed tool call should name ${TOOL_NAME}, got ${JSON.stringify(recorded[0].toolNames)}`,
+ ).toBe(true);
+ expect(
+ recorded[recorded.length - 1].contentChunkCount,
+ 'the post-tool invocation should stream the answer',
+ ).toBeGreaterThan(0);
+ } else {
+ const turns = fixtureTurns(FIXTURE);
+ expect(
+ turns.length,
+ 'the fixture should hold more than one invocation for this single turn',
+ ).toBeGreaterThan(1);
+ expect(
+ namesTool(turns[0].toolNames),
+ `the recorded tool call should name ${TOOL_NAME}, got ${JSON.stringify(turns[0].toolNames)}`,
+ ).toBe(true);
+
+ /** Replay drives the real tool node, so the tool ran again in this run
+ * rather than being replayed as recorded output. */
+ const toolResult = JSON.stringify(assistant[0].content ?? []);
+ expect(toolResult, 'the replayed tool call should carry its real result').toContain(FACT);
+
+ const ledger = readReplayLedger(FIXTURE);
+ expect(
+ ledger.invocationsConsumed,
+ 'replay should advance through every invocation of the turn',
+ ).toBe(ledger.invocationsTotal);
+ assertFixtureConsumed(FIXTURE);
+ }
+
+ expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0);
+ });
+});
diff --git a/e2e/specs/mock/model-replay.spec.ts b/e2e/specs/mock/model-replay.spec.ts
new file mode 100644
index 0000000000..03131cdfb9
--- /dev/null
+++ b/e2e/specs/mock/model-replay.spec.ts
@@ -0,0 +1,191 @@
+import { expect, test } from '@playwright/test';
+import type { TMessage } from 'librechat-data-provider';
+import {
+ assertFixtureConsumed,
+ fixtureTurns,
+ readReplayLedger,
+ removeFixture,
+} from './replay.helpers';
+import {
+ NEW_CHAT_PATH,
+ fetchJson,
+ messagesView,
+ getAccessToken,
+ selectMockEndpoint,
+ sendMessageAndWaitForCompletion,
+} from './helpers';
+
+/**
+ * Record-once/replay-forever coverage: one recorded real-provider conversation
+ * replays keylessly through the real createRun → registered replay provider →
+ * graph → SSE → persistence chain.
+ *
+ * Record (writes the fixture; needs a real provider key):
+ * E2E_MODEL_FIXTURES=record E2E_MODEL_FIXTURE_NAME=deepseek-two-turn \
+ * E2E_RECORD_PROVIDER_API_KEY=<key> \
+ * npx playwright test --config=e2e/playwright.config.mock.ts model-replay
+ *
+ * Replay (default, keyless): the same drive steps; the conversation binds to
+ * the committed fixture by prompt text, assistant turns must equal the
+ * recorded turns exactly, and the consumption ledger must drain completely.
+ */
+const COMMITTED_FIXTURE = 'deepseek-two-turn';
+const RECORDING = process.env.E2E_MODEL_FIXTURES === 'record';
+/**
+ * Record mode collects every replay spec, so each one records only the fixture
+ * it owns and stands down for the others. A spec must never write a fixture
+ * other than its own: two fixtures carrying the same prompts would make the
+ * server-side binding ambiguous and refuse both.
+ */
+const RECORDING_THIS_FIXTURE =
+ RECORDING && process.env.E2E_MODEL_FIXTURE_NAME === COMMITTED_FIXTURE;
+const FIXTURE = COMMITTED_FIXTURE;
+const RECORD_ENDPOINT = {
+ label: 'Replay Record Provider',
+ model: process.env.E2E_RECORD_PROVIDER_MODEL || 'deepseek-chat',
+};
+
+/**
+ * The closing prompt deliberately asks for prose: a one-token answer streams
+ * as a single content delta wrapped in empty initialization and usage frames,
+ * which cannot demonstrate incremental content streaming however many chunks
+ * the provider emits around it.
+ */
+const TURN_PROMPTS = [
+ 'Name the two prime numbers between 20 and 30, comma separated, and nothing else.',
+ 'In two short sentences, explain why the sum of those two primes is an even number. Begin with the word "Because".',
+];
+
+test.describe('recorded model fixture replay', () => {
+ test('a recorded conversation replays deterministically through the real pipeline', async ({
+ page,
+ }) => {
+ test.skip(
+ RECORDING && !RECORDING_THIS_FIXTURE,
+ `recording ${process.env.E2E_MODEL_FIXTURE_NAME}`,
+ );
+ test.setTimeout(180_000);
+ const pageErrors: string[] = [];
+ page.on('pageerror', (error) => pageErrors.push(error.message));
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10_000 });
+ if (RECORDING) {
+ /** Proof of a fresh write: the assertions below cannot be satisfied by a
+ * pre-existing fixture, so a run whose recorder never installed fails
+ * instead of greening against a stale artifact. */
+ removeFixture(FIXTURE);
+ await selectMockEndpoint(page, RECORD_ENDPOINT);
+ }
+
+ for (const prompt of TURN_PROMPTS) {
+ await sendMessageAndWaitForCompletion(page, prompt, { timeout: 90_000 });
+ }
+
+ const conversationId = /\/c\/([^/]+)/.exec(page.url())?.[1];
+ expect(conversationId, 'conversation should have a persisted id').toBeTruthy();
+ const token = await getAccessToken(page);
+ const messages = await fetchJson<TMessage[]>(
+ page,
+ `/api/messages/${encodeURIComponent(conversationId as string)}`,
+ token,
+ );
+ /** Agents-pipeline messages persist their text inside `content` parts;
+ * top-level `text` stays empty. */
+ const persistedText = (message: TMessage): string => {
+ if (message.text) {
+ return message.text;
+ }
+ return (message.content ?? [])
+ .map((part) => {
+ if (part?.type !== 'text') {
+ return '';
+ }
+ const text = (part as { text?: string | { value?: string } }).text;
+ return typeof text === 'string' ? text : (text?.value ?? '');
+ })
+ .join('');
+ };
+ const assistantTexts = messages
+ .filter((message) => message.isCreatedByUser === false)
+ .map(persistedText);
+ expect(assistantTexts).toHaveLength(TURN_PROMPTS.length);
+
+ if (RECORDING) {
+ /** The recorder rides LangChain token callbacks, which the provider
+ * stream dispatches without awaiting — the recording quiesces shortly
+ * AFTER the durable-completion barrier, so the record-mode harvest
+ * polls for the settled fixture instead of asserting a single read.
+ * Replay mode needs no such poll: the replaying generator finishes its
+ * ledger writes before the turn can persist. */
+ await expect
+ .poll(
+ () => {
+ try {
+ const settled = fixtureTurns(FIXTURE);
+ return (
+ settled.length === TURN_PROMPTS.length &&
+ settled.every(
+ (turn, index) =>
+ turn.userText === TURN_PROMPTS[index] &&
+ turn.finalText === assistantTexts[index],
+ )
+ );
+ } catch {
+ return false;
+ }
+ },
+ {
+ timeout: 15_000,
+ intervals: [250, 500, 1_000],
+ message: 'recorded fixture should quiesce with the persisted assistant turns',
+ },
+ )
+ .toBe(true);
+ const recorded = fixtureTurns(FIXTURE);
+ for (const turn of recorded) {
+ expect(turn.contentChunkCount, 'each turn should record assistant content').toBeGreaterThan(
+ 0,
+ );
+ }
+ expect(
+ recorded[recorded.length - 1].contentChunkCount,
+ 'the prose turn should record several content deltas, not one delta padded with empty frames',
+ ).toBeGreaterThan(1);
+ } else {
+ const turns = fixtureTurns(FIXTURE);
+ expect(
+ turns.map((turn) => turn.userText),
+ 'fixture invocations should mirror the driven prompts',
+ ).toEqual(TURN_PROMPTS);
+ expect(assistantTexts, 'replayed assistant turns should equal the recording exactly').toEqual(
+ turns.map((turn) => turn.finalText),
+ );
+ /** Compare against rendered markdown, not the raw recording: a reply
+ * opening with `52.` is rendered as an ordered-list marker and never
+ * appears in the DOM text, so a leading enumerator is stripped before
+ * matching and only a prose prefix is used. */
+ const renderedPrefix = turns[turns.length - 1].finalText
+ .replace(/^\s*\d+[.)]\s*/, '')
+ .trim()
+ .slice(0, 30);
+ expect(
+ renderedPrefix.length,
+ 'the prose turn should yield a comparable prefix',
+ ).toBeGreaterThan(15);
+ await expect(messagesView(page)).toContainText(renderedPrefix);
+
+ /** Streaming incrementality is asserted from the ledger's drained chunk
+ * count, not by sampling transient DOM — every recorded chunk passed
+ * through the live SSE wire before the durable completion barrier. */
+ const ledger = readReplayLedger(FIXTURE);
+ expect(ledger.chunksConsumed).toBe(ledger.chunksTotal);
+ expect(
+ turns[turns.length - 1].contentChunkCount,
+ 'replay should stream several content deltas for the prose turn',
+ ).toBeGreaterThan(1);
+ assertFixtureConsumed(FIXTURE);
+ }
+
+ expect(pageErrors, `Unexpected runtime errors: ${pageErrors.join(', ')}`).toHaveLength(0);
+ });
+});
diff --git a/e2e/specs/mock/replay.helpers.ts b/e2e/specs/mock/replay.helpers.ts
new file mode 100644
index 0000000000..042ded965b
--- /dev/null
+++ b/e2e/specs/mock/replay.helpers.ts
@@ -0,0 +1,145 @@
+import fs from 'fs';
+import path from 'path';
+
+/**
+ * Spec-side readers for the model-fixture replay lane. The server-side
+ * recorder/replayer (`e2e/setup/model-replay.js`) owns the formats; these
+ * readers stay dependency-free on that CJS module so the spec plane needs no
+ * runtime import of server code.
+ */
+const FIXTURES_DIR = path.resolve(__dirname, '../../fixtures/model-replay');
+const LEDGER_DIR = path.resolve(__dirname, '../.test-results/model-replay');
+
+export type FixtureTurn = {
+ userText: string;
+ finalText: string;
+ chunkCount: number;
+ /**
+ * Chunks carrying assistant text, as distinct from the empty
+ * initialization and usage-metadata chunks a provider also emits. Only
+ * these prove incremental content streaming — a total chunk count above one
+ * is satisfied by a single content delta wrapped in empty frames.
+ */
+ contentChunkCount: number;
+ /** Chunks carrying `tool_call_chunks`, i.e. the streamed tool invocation. */
+ toolCallChunkCount: number;
+ /** Tool names streamed by this invocation, in order of first appearance. */
+ toolNames: string[];
+};
+
+export type ReplayLedger = {
+ fixture: string;
+ invocationsTotal: number;
+ chunksTotal: number;
+ invocationsConsumed: number;
+ chunksConsumed: number;
+ overruns: Array<{ at: string; userText: string }>;
+ promptMismatches: Array<{ invocation: number; expected: string; received: string }>;
+};
+
+export function fixturePath(name: string): string {
+ return path.join(FIXTURES_DIR, `${name}.jsonl`);
+}
+
+/**
+ * Remove a fixture before a recording run so its assertions cannot be
+ * satisfied by a pre-existing artifact. Without this, a run whose hook failed
+ * to install the recorder would still see the live provider answer these
+ * deterministic prompts while the poll read the stale file — matching answers,
+ * valid chunk counts, and a green run that wrote nothing.
+ */
+export function removeFixture(name: string): void {
+ fs.rmSync(fixturePath(name), { force: true });
+}
+
+/**
+ * Parse a fixture's invocations in recorded order. An invocation's final text
+ * is the concatenation of its recorded chunk texts — the chunks are written
+ * synchronously during the stream, while the provider's `handleLLMEnd`
+ * dispatch (the `end` line) can land after the durable-completion barrier a
+ * spec waits on, so nothing here depends on it.
+ */
+export function fixtureTurns(name: string): FixtureTurn[] {
+ const lines = fs
+ .readFileSync(fixturePath(name), 'utf8')
+ .split('\n')
+ .filter(Boolean)
+ .map((line) => JSON.parse(line) as Record<string, unknown>);
+ const turns: FixtureTurn[] = [];
+ for (const entry of lines) {
+ if (entry.type === 'invocation') {
+ turns[entry.index as number] = {
+ userText: entry.userText as string,
+ finalText: '',
+ chunkCount: 0,
+ contentChunkCount: 0,
+ toolCallChunkCount: 0,
+ toolNames: [],
+ };
+ } else if (entry.type === 'chunk') {
+ const turn = turns[entry.invocation as number];
+ if (turn) {
+ const text = (entry.text as string) ?? '';
+ turn.chunkCount += 1;
+ turn.finalText += text;
+ if (text !== '') {
+ turn.contentChunkCount += 1;
+ }
+ const message = entry.message as
+ | { tool_call_chunks?: Array<{ name?: string }> }
+ | undefined;
+ const toolCallChunks = message?.tool_call_chunks ?? [];
+ if (toolCallChunks.length > 0) {
+ turn.toolCallChunkCount += 1;
+ for (const call of toolCallChunks) {
+ if (call.name && !turn.toolNames.includes(call.name)) {
+ turn.toolNames.push(call.name);
+ }
+ }
+ }
+ }
+ } else if (entry.type === 'error') {
+ throw new Error(`Fixture ${name} recorded a provider error: ${String(entry.message)}`);
+ }
+ }
+ return turns;
+}
+
+export function readReplayLedger(name: string): ReplayLedger {
+ const ledgerPath = path.join(LEDGER_DIR, `${name}.json`);
+ if (!fs.existsSync(ledgerPath)) {
+ throw new Error(
+ `Replay ledger missing for fixture "${name}" (${ledgerPath}); ` +
+ 'the conversation never bound to the fixture',
+ );
+ }
+ return JSON.parse(fs.readFileSync(ledgerPath, 'utf8')) as ReplayLedger;
+}
+
+/**
+ * The teardown consumption check: every recorded invocation and chunk was
+ * drained, nothing was invoked past the script, and every prompt matched its
+ * recording. Converts silent underruns and shifted bindings into crisp
+ * diagnostics.
+ */
+export function assertFixtureConsumed(name: string): void {
+ const ledger = readReplayLedger(name);
+ const failures: string[] = [];
+ if (ledger.invocationsConsumed !== ledger.invocationsTotal) {
+ failures.push(
+ `under-consumed: ${ledger.invocationsConsumed}/${ledger.invocationsTotal} invocations`,
+ );
+ }
+ if (ledger.chunksConsumed !== ledger.chunksTotal) {
+ failures.push(`under-streamed: ${ledger.chunksConsumed}/${ledger.chunksTotal} chunks`);
+ }
+ if (ledger.overruns.length > 0) {
+ failures.push(`over-consumed ${ledger.overruns.length}x: ${JSON.stringify(ledger.overruns)}`);
+ }
+ if (ledger.promptMismatches.length > 0) {
+ failures.push(`prompt mismatches: ${JSON.stringify(ledger.promptMismatches)}`);
+ }
+ if (failures.length > 0) {
+ throw new Error(`Fixture "${name}" consumption check failed — ${failures.join('; ')}`);
+ }
+}
diff --git a/packages/api/src/agents/run.ts b/packages/api/src/agents/run.ts
index 13b2725986..029807ebba 100644
--- a/packages/api/src/agents/run.ts
+++ b/packages/api/src/agents/run.ts
@@ -1301,6 +1301,7 @@ function buildSubagentConfigs(
export async function createRun({
runId,
signal,
+ conversationId,
agents,
messages,
discoveredToolNames,
@@ -1331,6 +1332,9 @@ export async function createRun({
agents: RunAgent[];
signal: AbortSignal;
runId?: string;
+ /** Conversation-stable identity, used by the e2e run hook to tell a resumed
+ * run apart from a fresh attempt (a resume carries no messages). */
+ conversationId?: string;
streaming?: boolean;
streamUsage?: boolean;
requestBody?: t.RequestBody;
@@ -1991,6 +1995,11 @@ export async function createRun({
const run = await Run.create(runConfig);
applyCustomHandoffPromptKeyCompatibility(run, runConfig.graphConfig);
- applyTestRunHook(run, { messages, agents, modelCallbacks });
+ applyTestRunHook(run, {
+ messages,
+ agents,
+ modelCallbacks,
+ conversationId: conversationId ?? requestBody?.conversationId ?? undefined,
+ });
return run;
}
diff --git a/packages/api/src/agents/testHook.ts b/packages/api/src/agents/testHook.ts
index ef0a83d968..487369aa86 100644
--- a/packages/api/src/agents/testHook.ts
+++ b/packages/api/src/agents/testHook.ts
@@ -9,6 +9,13 @@ import type { ModelBoundChatModelCallback } from '~/middleware/modelBoundContent
*/
export interface TestRunHookContext {
messages?: BaseMessage[];
+ /**
+ * Identifies the conversation this run belongs to. A resumed run rebuilds
+ * `createRun` with no messages because state is rehydrated from the
+ * checkpoint, so message history alone cannot tell a resume apart from a
+ * fresh attempt; this is the stable identity across both.
+ */
+ conversationId?: string;
agents: ReadonlyArray<{ tools?: ReadonlyArray<{ name: string }> }>;
modelCallbacks?: readonly ModelBoundChatModelCallback[];
}