RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(benchmark): account for failed requests and incomplete streams (#19498)
JiaLin Ning03ca271f73de
diff --git a/test/benchmark/README.md b/test/benchmark/README.mdindex c21bce7bba..baa631affe 100644--- a/test/benchmark/README.md+++ b/test/benchmark/README.md@@ -165,6 +165,56 @@ What this CLI can do- It is not a general admin CLI; there are no standalone "create-only" or"manage" commands. Use the reports to capture created IDs for reuse.+## Interpreting failures and throughput++Each measured request contributes one success or failure, including when using+`--concurrency`. Connection failures, timeouts, non-2xx HTTP responses, invalid+response payloads, and API errors are recorded as failed samples so the remaining+iterations can run and produce a report. There are no automatic retries: a retry+would hide the original failure and change the measured workload. Authentication+and dataset/chat setup are outside the measured requests; setup failures still+stop the command.++A successful chat sample requires non-empty assistant content **and** a stream+completion marker (`[DONE]` or a non-empty `finish_reason`). Receiving some text+before an unexpected EOF is a failed, partial response. Partial text remains+available with `--print-response`, but failed samples are excluded from both+first-token and total-latency statistics. Completion measures protocol success,+not answer quality: a `length` finish reason can still indicate an answer limited+by the model's token budget. An empty retrieval result with `code: 0` is a+successful API request, not evidence of good retrieval quality.++Reports distinguish:++| JSON field | Definition |+| --- | --- |+| `qps` | All measured requests / measured wall-clock duration |+| `success_qps` | Successful requests / measured wall-clock duration |+| `failure_rate` | Failed requests / all measured requests, in the range 0–1 |++For example, if 10 measured requests finish in 5 seconds but only 6 succeed,+`qps` is 2, `success_qps` is 1.2, and `failure_rate` is 0.4. Comparing only QPS+can make a server that quickly rejects requests look faster than a healthy one.+The text report shows the same distinction and renders failure rate as a+percentage. If all requests fail, successful QPS is zero and successful latency+statistics are unavailable (`null` in JSON / `n/a` in text).++The command exits with status 1 if any measured request fails and 0 if all+measured requests succeed. Unexpected programming errors are not converted into+request failures.++### Testing the benchmark client++From the repository root, with the test dependencies installed:++```sh+uv run --group test pytest test/benchmark/tests+```++These tests use response fixtures and a loopback HTTP server. They also invoke+the actual CLI with sequential and multiprocessing workloads. No RAGFlow server,+database, model, or external credentials are required.+Do I need the dataset ID?- If the CLI creates a dataset, it uses the returned dataset ID internally.You do not need to supply it for that same run.diff --git a/test/benchmark/chat.py b/test/benchmark/chat.pyindex 7d38ebc000..1c21b23f59 100644--- a/test/benchmark/chat.py+++ b/test/benchmark/chat.py@@ -2,6 +2,8 @@ import jsonimport timefrom typing import Any, Dict, List, Optional+import requests+from .http_client import HttpClientfrom .metrics import ChatSample@@ -55,15 +57,18 @@ def resolve_model(model: Optional[str], chat_data: Optional[Dict[str, Any]]) ->def _parse_stream_error(response) -> Optional[str]:+ """Reject HTTP failures and responses that cannot contain a chat stream."""+ if not 200 <= response.status_code < 300:+ return f"HTTP {response.status_code}"content_type = response.headers.get("Content-Type", "")if "text/event-stream" in content_type:return Nonetry:payload = response.json()- except Exception:+ except ValueError:return f"Unexpected non-stream response (status {response.status_code})"- if payload.get("code") not in (0, None):- return payload.get("message", "Unknown error")+ if isinstance(payload, dict) and payload.get("code") not in (0, None):+ return str(payload.get("message") or f"Chat failed (code {payload['code']})")return f"Unexpected non-stream response (status {response.status_code})"@@ -74,26 +79,27 @@ def stream_chat_completion(messages: List[Dict[str, Any]],extra_body: Optional[Dict[str, Any]] = None,) -> ChatSample:+ """Measure one text completion, retaining partial text when the stream fails."""payload: Dict[str, Any] = {"model": model, "messages": messages, "stream": True}if extra_body:payload["extra_body"] = extra_bodyt0 = time.perf_counter()- response = client.request(- "POST",- f"/openai/{chat_id}/chat/completions",- json_body=payload,- stream=True,- )- error = _parse_stream_error(response)- if error:- response.close()- return ChatSample(t0=t0, t1=None, t2=None, error=error)-+ response = Nonet1: Optional[float] = Nonet2: Optional[float] = Nonestream_error: Optional[str] = None+ completed = Falsecontent_parts: List[str] = []try:+ response = client.request(+ "POST",+ f"/openai/{chat_id}/chat/completions",+ json_body=payload,+ stream=True,+ )+ error = _parse_stream_error(response)+ if error:+ return ChatSample(t0=t0, t1=None, t2=time.perf_counter(), error=error)for raw_line in response.iter_lines(decode_unicode=True):if raw_line is None:continue@@ -104,17 +110,33 @@ def stream_chat_completion(if not data:continueif data == "[DONE]":+ completed = Truet2 = time.perf_counter()breaktry:chunk = json.loads(data)- except Exception as exc:+ except ValueError as exc:stream_error = f"Invalid JSON chunk: {exc}"t2 = time.perf_counter()break- choices = chunk.get("choices") or []- choice = choices[0] if choices else {}- delta = choice.get("delta") or {}+ if not isinstance(chunk, dict):+ stream_error = "Invalid stream chunk: expected an object"+ break+ if chunk.get("error") is not None or chunk.get("code") not in (0, None):+ stream_error = "Chat stream reported an error"+ break+ choices = chunk.get("choices")+ if not isinstance(choices, list) or not all(isinstance(item, dict) for item in choices):+ stream_error = "Invalid stream chunk: expected choices to be a list of objects"+ break+ # Usage-only chunks can have no choices; they do not complete the stream.+ if not choices:+ continue+ if not all(isinstance(item.get("delta"), dict) for item in choices):+ stream_error = "Invalid stream chunk: expected delta to be an object"+ break+ choice = choices[0]+ delta = choice["delta"]content = delta.get("content")if t1 is None and isinstance(content, str) and content != "":t1 = time.perf_counter()@@ -122,10 +144,14 @@ def stream_chat_completion(content_parts.append(content)finish_reason = choice.get("finish_reason")if finish_reason:+ completed = Truet2 = time.perf_counter()break+ except requests.RequestException as exc:+ stream_error = f"Transport error ({type(exc).__name__})"finally:- response.close()+ if response is not None:+ response.close()if t2 is None:t2 = time.perf_counter()@@ -134,4 +160,6 @@ def stream_chat_completion(return ChatSample(t0=t0, t1=t1, t2=t2, error=stream_error, response_text=response_text)if t1 is None:return ChatSample(t0=t0, t1=None, t2=t2, error="No assistant content received", response_text=response_text)+ if not completed:+ return ChatSample(t0=t0, t1=t1, t2=t2, error="Stream ended before a completion marker", response_text=response_text)return ChatSample(t0=t0, t1=t1, t2=t2, error=None, response_text=response_text)diff --git a/test/benchmark/cli.py b/test/benchmark/cli.pyindex e0e9352451..46dfd882f8 100644--- a/test/benchmark/cli.py+++ b/test/benchmark/cli.py@@ -22,7 +22,7 @@ from .dataset import (wait_for_parse_done,)from .http_client import HttpClient-from .metrics import ChatSample, RetrievalSample, summarize+from .metrics import ChatSample, RetrievalSample, request_rates, summarizefrom .report import chat_report, retrieval_reportfrom .retrieval import RetrievalError, build_payload, run_retrieval as run_retrieval_requestfrom .utils import eprint, load_json_arg, split_csv@@ -457,7 +457,7 @@ def run_chat(client: HttpClient, args: argparse.Namespace) -> int:"errors": [e for e in errors if e],"created": created,"total_duration_s": total_duration,- "qps": (args.iterations / total_duration) if total_duration > 0 else None,+ **request_rates(success, failure, total_duration),}if args.print_response:payload["responses"] = responses@@ -557,7 +557,7 @@ def run_retrieval(client: HttpClient, args: argparse.Namespace) -> int:"errors": [e for e in errors if e],"created": created,"total_duration_s": total_duration,- "qps": (args.iterations / total_duration) if total_duration > 0 else None,+ **request_rates(success, failure, total_duration),}if args.print_response:payload["responses"] = responsesdiff --git a/test/benchmark/metrics.py b/test/benchmark/metrics.pyindex 02183ec493..6b7477d70d 100644--- a/test/benchmark/metrics.py+++ b/test/benchmark/metrics.py@@ -65,3 +65,14 @@ def summarize(values: List[float]) -> dict:"p90": _percentile(sorted_vals, 90),"p95": _percentile(sorted_vals, 95),}+++def request_rates(success: int, failure: int, total_duration_s: float | None) -> dict:+ """Separate attempted throughput from successfully completed work."""+ total = success + failure+ timed = total_duration_s is not None and total_duration_s > 0+ return {+ "qps": total / total_duration_s if timed else None,+ "success_qps": success / total_duration_s if timed else None,+ "failure_rate": failure / total if total else None,+ }diff --git a/test/benchmark/report.py b/test/benchmark/report.pyindex 12a16cf18a..d1525f1fcc 100644--- a/test/benchmark/report.py+++ b/test/benchmark/report.py@@ -1,5 +1,7 @@from typing import Dict, List, Optional+from .metrics import request_rates+def _fmt_seconds(value: Optional[float]) -> str:if value is None:@@ -14,15 +16,19 @@ def _fmt_ms(value: Optional[float]) -> str:def _fmt_qps(qps: Optional[float]) -> str:- if qps is None or qps <= 0:+ if qps is None:return "n/a"return f"{qps:.2f}"-def _calc_qps(total_duration_s: Optional[float], total_requests: int) -> Optional[float]:- if total_duration_s is None or total_duration_s <= 0:- return None- return total_requests / total_duration_s+def _request_rate_lines(success: int, failure: int, total_duration_s: float | None) -> list[str]:+ rates = request_rates(success, failure, total_duration_s)+ failure_rate = "n/a" if rates["failure_rate"] is None else f"{rates['failure_rate']:.2%}"+ return [+ f"QPS (requests / total duration): {_fmt_qps(rates['qps'])}",+ f"Successful QPS (success / total duration): {_fmt_qps(rates['success_qps'])}",+ f"Failure Rate: {failure_rate}",+ ]def render_report(lines: List[str]) -> str:@@ -62,7 +68,7 @@ def chat_report(f"avg={_fmt_ms(first_token_stats['avg'])}, min={_fmt_ms(first_token_stats['min'])}, "f"p50={_fmt_ms(first_token_stats['p50'])}, p90={_fmt_ms(first_token_stats['p90'])}, p95={_fmt_ms(first_token_stats['p95'])}",f"Total Duration: {_fmt_seconds(total_duration_s)}",- f"QPS (requests / total duration): {_fmt_qps(_calc_qps(total_duration_s, iterations))}",+ *_request_rate_lines(success, failure, total_duration_s),])if errors:@@ -95,7 +101,7 @@ def retrieval_report([f"Latency: avg={_fmt_ms(stats['avg'])}, min={_fmt_ms(stats['min'])}, p50={_fmt_ms(stats['p50'])}, p90={_fmt_ms(stats['p90'])}, p95={_fmt_ms(stats['p95'])}",f"Total Duration: {_fmt_seconds(total_duration_s)}",- f"QPS (requests / total duration): {_fmt_qps(_calc_qps(total_duration_s, iterations))}",+ *_request_rate_lines(success, failure, total_duration_s),])if errors:diff --git a/test/benchmark/retrieval.py b/test/benchmark/retrieval.pyindex c2a4880041..f824ebdc39 100644--- a/test/benchmark/retrieval.py+++ b/test/benchmark/retrieval.py@@ -1,6 +1,8 @@import timefrom typing import Any, Dict, List, Optional+import requests+from .http_client import HttpClientfrom .metrics import RetrievalSample@@ -26,14 +28,27 @@ def build_payload(def run_retrieval(client: HttpClient, payload: Dict[str, Any]) -> RetrievalSample:+ """Measure one retrieval, recording transport and API failures as samples."""t0 = time.perf_counter()- response = client.request("POST", "/retrieval", json_body=payload, stream=False)- raw = response.content- t1 = time.perf_counter()+ response = Nonetry:- res = client.parse_json_bytes(raw)- except Exception as exc:- return RetrievalSample(t0=t0, t1=t1, error=f"Invalid JSON response: {exc}")+ response = client.request("POST", "/retrieval", json_body=payload, stream=False)+ raw = response.content+ t1 = time.perf_counter()+ if not 200 <= response.status_code < 300:+ return RetrievalSample(t0=t0, t1=t1, error=f"HTTP {response.status_code}")+ try:+ res = client.parse_json_bytes(raw)+ except ValueError as exc:+ return RetrievalSample(t0=t0, t1=t1, error=f"Invalid JSON response: {exc}")+ except requests.RequestException as exc:+ return RetrievalSample(t0=t0, t1=time.perf_counter(), error=f"Transport error ({type(exc).__name__})")+ finally:+ if response is not None:+ response.close()+ if not isinstance(res, dict):+ return RetrievalSample(t0=t0, t1=t1, error="Invalid retrieval response: expected an object")if res.get("code") != 0:- return RetrievalSample(t0=t0, t1=t1, error=res.get("message"), response=res)+ error = str(res.get("message") or f"Retrieval failed (code {res.get('code')!r})")+ return RetrievalSample(t0=t0, t1=t1, error=error, response=res)return RetrievalSample(t0=t0, t1=t1, error=None, response=res)diff --git a/test/benchmark/tests/test_cli_failures.py b/test/benchmark/tests/test_cli_failures.pynew file mode 100644index 0000000000..1f5b109cf9--- /dev/null+++ b/test/benchmark/tests/test_cli_failures.py@@ -0,0 +1,100 @@+"""Exercise the real CLI and process workers against a loopback HTTP fixture."""++import json+import os+import subprocess+import sys+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path++import pytest+++def benchmark_server():+ class Handler(BaseHTTPRequestHandler):+ def log_message(self, *args):+ pass++ def send_body(self, body, status=200, content_type="application/json"):+ self.send_response(status)+ self.send_header("Content-Type", content_type)+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def do_GET(self):+ self.send_body(b'{"code":0,"data":{"id":"chat","llm_id":"fixture-model"}}')++ def do_POST(self):+ self.rfile.read(int(self.headers.get("Content-Length", "0")))+ with self.server.lock:+ index = self.server.requests+ self.server.requests += 1+ if index == 4:+ # Close without response headers: requests raises ConnectionError.+ self.close_connection = True+ return+ if self.path.endswith("/retrieval"):+ bodies = [b'{"code":0,"data":{"chunks":[]}}', b'{"code":102}', b'{"code":0}', b"[]"]+ self.send_body(bodies[index], status=503 if index == 2 else 200)+ else:+ content = b'data: {"choices":[{"delta":{"content":"answer"}}]}\n\n'+ bodies = [content + b"data: [DONE]\n\n", content, content + b"data: [DONE]\n\n", content + b'data: {"error":{"message":"failed"}}\n\n']+ self.send_body(bodies[index], status=503 if index == 2 else 200, content_type="text/event-stream; charset=utf-8")++ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)+ server.requests = 0+ server.lock = threading.Lock()+ worker = threading.Thread(target=server.serve_forever, daemon=True)+ worker.start()+ try:+ yield server+ finally:+ server.shutdown()+ worker.join()+ server.server_close()++[email protected]("command", ["chat", "retrieval"])[email protected]("concurrency", [1, 2])+def test_cli_finishes_mixed_workload_and_reports_only_successful_latency(benchmark_server, command, concurrency):+ repo = Path(__file__).resolve().parents[3]+ args = [+ sys.executable,+ "-m",+ "benchmark",+ command,+ "--base-url",+ f"http://127.0.0.1:{benchmark_server.server_port}",+ "--api-key",+ "local-fixture-token",+ "--iterations",+ "5",+ "--concurrency",+ str(concurrency),+ "--json",+ "--print-response",+ ]+ if command == "chat":+ args += ["--chat-id", "chat", "--model", "fixture-model", "--message", "hello"]+ else:+ args += ["--dataset-id", "dataset", "--question", "hello"]+ env = {**os.environ, "PYTHONPATH": str(repo / "test"), "NO_PROXY": "127.0.0.1", "no_proxy": "127.0.0.1"}+ result = subprocess.run(args, cwd=repo, env=env, capture_output=True, text=True, timeout=30, check=False)+ assert result.returncode == 1, result.stderr+ report = json.loads(result.stdout)+ assert benchmark_server.requests == 5+ assert report["success"] == 1+ assert report["failure"] == 4+ assert report["failure_rate"] == 0.8+ assert report["success_qps"] == pytest.approx(report["qps"] / 5)+ assert len(report["errors"]) == 4+ assert len(report["responses"]) == 5+ latency = "total_latency" if command == "chat" else "latency"+ assert report[latency]["count"] == 1+ if command == "chat":+ assert report["first_token_latency"]["count"] == 1+ assert any("[error]" in response and "answer" in response for response in report["responses"])+ assert "Traceback" not in result.stderrdiff --git a/test/benchmark/tests/test_reports.py b/test/benchmark/tests/test_reports.pynew file mode 100644index 0000000000..d7ed2d50f6--- /dev/null+++ b/test/benchmark/tests/test_reports.py@@ -0,0 +1,30 @@+import pytest++from test.benchmark.metrics import request_rates, summarize+from test.benchmark.report import chat_report, retrieval_report+++def test_request_rates_separate_attempts_and_successes():+ assert request_rates(2, 3, 5) == {"qps": 1.0, "success_qps": 0.4, "failure_rate": 0.6}+++def test_zero_success_is_zero_throughput_not_unavailable():+ assert request_rates(0, 4, 2) == {"qps": 2.0, "success_qps": 0.0, "failure_rate": 1.0}++[email protected]("duration", [None, 0])+def test_missing_duration_does_not_invent_throughput(duration):+ assert request_rates(0, 0, duration) == {"qps": None, "success_qps": None, "failure_rate": None}++[email protected]("command", ["chat", "retrieval"])+def test_text_report_shows_all_failed_run(command):+ kwargs = {"interface": command, "concurrency": 2, "total_duration_s": 2, "iterations": 4, "success": 0, "failure": 4, "errors": ["HTTP 503"], "created": {}}+ stats = summarize([])+ if command == "chat":+ report = chat_report(**kwargs, model="model", total_stats=stats, first_token_stats=stats)+ else:+ report = retrieval_report(**kwargs, stats=stats)+ assert "Successful QPS (success / total duration): 0.00" in report+ assert "Failure Rate: 100.00%" in report+ assert "avg=n/a" in reportdiff --git a/test/benchmark/tests/test_samples.py b/test/benchmark/tests/test_samples.pynew file mode 100644index 0000000000..fa72b00526--- /dev/null+++ b/test/benchmark/tests/test_samples.py@@ -0,0 +1,190 @@+"""Request failures must become samples, not successful or missing measurements."""++import json+from unittest.mock import Mock++import pytest+import requests++from test.benchmark.chat import stream_chat_completion+from test.benchmark.http_client import HttpClient+from test.benchmark.retrieval import run_retrieval+++def stream_response(events, status=200):+ response = Mock(spec=requests.Response)+ response.status_code = status+ response.headers = {"Content-Type": "text/event-stream"}++ def lines(**kwargs):+ for event in events:+ if isinstance(event, Exception):+ raise event+ yield "data: " + (event if isinstance(event, str) else json.dumps(event))++ response.iter_lines.side_effect = lines+ return response+++CONTENT = {"choices": [{"delta": {"content": "partial answer"}}]}+++def test_chat_eof_after_content_is_failure():+ client = Mock(spec=HttpClient)+ client.request.return_value = stream_response([CONTENT])+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error+ assert sample.response_text == "partial answer"+ client.request.return_value.close.assert_called_once()++[email protected]("failure", [requests.ConnectTimeout(), requests.ConnectionError()])+def test_connection_failure_returns_chat_sample(failure):+ client = Mock(spec=HttpClient)+ client.request.side_effect = failure+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error+ assert sample.first_token_latency is None+++def test_midstream_timeout_retains_partial_response():+ client = Mock(spec=HttpClient)+ client.request.return_value = stream_response([CONTENT, requests.ReadTimeout()])+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error+ assert sample.response_text == "partial answer"+ assert sample.first_token_latency is not None+ client.request.return_value.close.assert_called_once()++[email protected]("failure", [requests.ReadTimeout(), requests.ConnectionError()])+def test_connection_failure_returns_retrieval_sample(failure):+ client = Mock(spec=HttpClient)+ client.request.side_effect = failure+ sample = run_retrieval(client, {"question": "question", "dataset_ids": ["dataset"]})+ assert sample.error+++def test_retrieval_error_without_message_is_failure():+ response = requests.Response()+ response.status_code = 200+ response._content = b'{"code": 102}'+ client = HttpClient("http://unused.invalid")+ client.request = Mock(return_value=response)+ sample = run_retrieval(client, {})+ assert sample.error++[email protected]("ending", ["[DONE]", {"choices": [{"delta": {}, "finish_reason": "stop"}]}])+def test_completed_chat_remains_successful(ending):+ client = Mock(spec=HttpClient)+ client.request.return_value = stream_response([CONTENT, ending])+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error is None+ assert sample.response_text == "partial answer"+ assert sample.total_latency >= sample.first_token_latency >= 0+ client.request.return_value.close.assert_called_once()+++ "event",+ [+ "not-json",+ [],+ None,+ {},+ *({"choices": value} for value in [None, False, 0, "", {}, [None], [{}, "invalid"]]),+ {"choices": {"delta": {}}},+ *({"choices": [{"delta": value}]} for value in [None, False, 0, "", [], "bad"]),+ {"choices": [{}]},+ {"choices": [{"delta": {}}, {"delta": None}]},+ {"error": {"message": "unavailable"}},+ {"code": 102},+ ],+)+def test_malformed_or_error_event_after_content_is_failure(event):+ client = Mock(spec=HttpClient)+ client.request.return_value = stream_response([CONTENT, event, "[DONE]"])+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error+ assert sample.response_text == "partial answer"+ client.request.return_value.close.assert_called_once()+++def test_chat_http_error_cannot_be_hidden_by_valid_sse_body():+ client = Mock(spec=HttpClient)+ response = stream_response([CONTENT, "[DONE]"], status=503)+ client.request.return_value = response+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error == "HTTP 503"+ response.iter_lines.assert_not_called()+ response.close.assert_called_once()++[email protected]("payload", [[], None, {"code": 102}, {"code": 102, "message": ""}])+def test_non_stream_chat_errors_do_not_escape(payload):+ client = Mock(spec=HttpClient)+ response = Mock(spec=requests.Response)+ response.status_code = 200+ response.headers = {"Content-Type": "application/json"}+ response.json.return_value = payload+ client.request.return_value = response+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error+ response.close.assert_called_once()++[email protected]("status,payload", [(503, {"code": 0}), (200, []), (200, None), (200, {}), (200, {"code": 102, "message": ""})])+def test_invalid_retrieval_response_is_failure(status, payload):+ response = requests.Response()+ response.status_code = status+ response._content = json.dumps(payload).encode()+ client = HttpClient("http://unused.invalid")+ client.request = Mock(return_value=response)+ sample = run_retrieval(client, {})+ assert sample.error+++def test_successful_empty_retrieval_is_not_a_transport_failure():+ response = requests.Response()+ response.status_code = 200+ response._content = b'{"code": 0, "data": {"chunks": []}}'+ client = HttpClient("http://unused.invalid")+ client.request = Mock(return_value=response)+ sample = run_retrieval(client, {})+ assert sample.error is None+ assert sample.response["data"]["chunks"] == []+++def test_empty_completed_chat_still_fails():+ client = Mock(spec=HttpClient)+ client.request.return_value = stream_response(["[DONE]"])+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error == "No assistant content received"+++def test_programming_errors_are_not_swallowed():+ client = Mock(spec=HttpClient)+ client.request.side_effect = RuntimeError("unexpected bug")+ with pytest.raises(RuntimeError, match="unexpected bug"):+ stream_chat_completion(client, "chat", "model", [])+ with pytest.raises(RuntimeError, match="unexpected bug"):+ run_retrieval(client, {})+++ "event",+ [+ {"choices": [], "usage": {"total_tokens": 4}},+ {"choices": [{"delta": {"role": "assistant", "content": None}}]},+ {"choices": [{"delta": {}}, {"delta": {}}]},+ ],+)+def test_valid_metadata_chunks_preserve_success(event):+ """Usage-only and empty deltas must not invalidate completed text streams."""+ client = Mock(spec=HttpClient)+ client.request.return_value = stream_response([CONTENT, event, "[DONE]"])+ sample = stream_chat_completion(client, "chat", "model", [])+ assert sample.error is None+ assert sample.response_text == "partial answer"+ client.request.return_value.close.assert_called_once()