~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
test(perf): assert chat context scaling in bounded CI (#8008)
LearningCircuit49837388e9df
diff --git a/.github/workflows/performance-assertions.yml b/.github/workflows/performance-assertions.ymlnew file mode 100644index 0000000000..36404dfe10--- /dev/null+++ b/.github/workflows/performance-assertions.yml@@ -0,0 +1,84 @@+name: Performance Assertions++on:+ pull_request:+ paths:+ - 'src/**'+ - 'tests/performance/**'+ - 'tests/search_quality/**'+ - 'tests/ci/test_offline_workload_shards.py'+ - 'tests/ci/standalone/test_performance_pytest.py'+ - 'scripts/ci/performance_pytest.py'+ - 'scripts/ci/rag_index_workload.py'+ - 'scripts/ci/rag_vector_scale_benchmark.py'+ - '.github/workflows/performance-assertions.yml'+ - 'pyproject.toml'+ - 'pdm.lock'+ push:+ branches: [main]+ paths:+ - 'src/**'+ - 'tests/performance/**'+ - 'tests/search_quality/**'+ - 'tests/ci/test_offline_workload_shards.py'+ - 'tests/ci/standalone/test_performance_pytest.py'+ - 'scripts/ci/performance_pytest.py'+ - 'scripts/ci/rag_index_workload.py'+ - 'scripts/ci/rag_vector_scale_benchmark.py'+ - '.github/workflows/performance-assertions.yml'+ - 'pyproject.toml'+ - 'pdm.lock'+ workflow_dispatch:++permissions: {}++jobs:+ assertions:+ name: Offline performance assertions+ runs-on: ubuntu-latest+ timeout-minutes: 30+ permissions:+ contents: read+ env:+ OMP_NUM_THREADS: '1'+ OPENBLAS_NUM_THREADS: '1'+ MKL_NUM_THREADS: '1'+ HF_HUB_OFFLINE: '1'+ TRANSFORMERS_OFFLINE: '1'+ TOKENIZERS_PARALLELISM: 'false'+ PYTHONOPTIMIZE: '0'+ steps:+ - name: Harden runner+ uses: step-security/harden-runner@dc2769abcc0fcfbadfc1df54081c1608579ef6b2 # v2.22.2+ with:+ egress-policy: audit+ - name: Checkout+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false+ - name: Set up PDM+ uses: pdm-project/setup-pdm@544d7237314ee09c256785bd360f6b30add38b37 # v4.6+ with:+ python-version: '3.12'+ version: 2.29.3+ cache: true+ cache-restore-exact-match: true+ - name: Install SQLCipher+ run: |+ sudo apt-get update+ sudo apt-get install -y libsqlcipher-dev+ - name: Check locked dependencies+ run: pdm lock --check+ - name: Install dependencies+ run: pdm sync -d --clean+ - name: Verify benchmark CI result handling+ run: pdm run python -m unittest discover -s tests/ci/standalone -p test_performance_pytest.py -v+ - name: Run bounded CI profiles and require assertions+ run: pdm run python scripts/ci/performance_pytest.py --profile ci --output test-results/performance+ - name: Upload asserted results and numeric measurements+ if: always()+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2+ with:+ name: performance-assertions+ path: test-results/performance+ retention-days: 14diff --git a/changelog.d/+chat-retained-findings.bugfix.md b/changelog.d/+chat-retained-findings.bugfix.mdnew file mode 100644index 0000000000..8fc30ea6d7--- /dev/null+++ b/changelog.d/+chat-retained-findings.bugfix.md@@ -0,0 +1 @@+Limit raw chat-context citation parsing to the retained findings window, avoiding repeated processing of discarded conversation history.diff --git a/docs/ci/workflow-status.md b/docs/ci/workflow-status.mdindex dc0578472b..060dc7dc58 100644--- a/docs/ci/workflow-status.md+++ b/docs/ci/workflow-status.md@@ -204,3 +204,22 @@ the table above.`release.yml` — on pushes to `main` or `v*.*.*` tags and on manual releasedispatches, gated on `should_release` (a `main` push whose version alreadyhas a release skips them) — or through this workflow's own manual dispatch.++## Offline performance assertions++`performance-assertions.yml` runs the available offline benchmark suites in a+serial, bounded CI profile on relevant pull requests and main-branch changes.+It explicitly enables integration checks and rejects missing, failed or skipped+JUnit evidence and unverified measurement reports. When present, the original+synthetic-PDF and vector-scale smoke cases cover flat and HNSW; larger release+and manual workloads retain their existing profiles.++Each invocation stores results in a fresh `run-*` directory under its output+path and updates `ci-summary.json` with the current run directory. Reports must+match the requested area/profile and contain finite samples consistent with+their counts and summaries. Existing manual-shard contracts join the selection+when available. Native database, HTTP and RAG smoke measurements and retrieval+quality reports are copied out of pytest temporary storage into the uploaded+`native/` results; database/RAG/HTTP use their fixed, asserted smoke settings,+recorded in each report. Use their standalone CLIs for larger local workloads.+Only numeric JSON and JUnit evidence is retained, rather than temporary databases.diff --git a/scripts/ci/performance_pytest.py b/scripts/ci/performance_pytest.pynew file mode 100644index 0000000000..b83deb7527--- /dev/null+++ b/scripts/ci/performance_pytest.py@@ -0,0 +1,381 @@+"""Run available offline benchmarks serially and require asserted results."""++import argparse+import json+import math+import os+from pathlib import Path+import statistics+import subprocess+import sys+import tempfile++from defusedxml import ElementTree as ET+++SUITES = (+ ("tests/performance/content_extraction", "extraction"),+ ("tests/performance/content_cache", "cache"),+ ("tests/performance/citation_formatting", "citations"),+ ("tests/performance/citation_metadata", "citenorm"),+ ("tests/performance/url_processing", "urls"),+ ("tests/performance/text_splitting", "splitting"),+ ("tests/performance/source_rendering", "sources"),+ ("tests/performance/json_extraction", "jsonparse"),+ ("tests/performance/chat_context", "chatctx"),+ ("tests/performance/token_tracking", "tokens"),+ ("tests/performance/source_persistence", "sourcepersist"),+ ("tests/performance/rag_scale_smoke", None),+ ("tests/performance/database/test_connection_workload.py", None),+ ("tests/performance/http/test_request_workload.py", None),+ ("tests/performance/rag/test_storage_workload.py", None),+ ("tests/search_quality", None),+)++NATIVE_REPORTS = {+ "tests/performance/database/test_connection_workload.py": (+ "database_performance",+ (),+ ),+ "tests/performance/http/test_request_workload.py": ("http_performance", ()),+ "tests/performance/rag/test_storage_workload.py": (+ "rag_performance",+ ("flat", "hnsw"),+ ),+ "tests/search_quality": ("search_quality", ("flat", "hnsw")),+}+++def available_suites(root):+ suites = [+ (target, area)+ for target, area in SUITES+ if (root / target).is_file()+ or (+ (root / target).is_dir() and any((root / target).rglob("test_*.py"))+ )+ ]+ if not suites:+ raise ValueError("No offline benchmark suites are present")+ targets = {target for target, _ in suites}+ if "tests/performance/citation_metadata" in targets:+ suites.append(("tests/utilities/test_citation_normalizer.py", None))+ if "tests/performance/rag_scale_smoke" in targets:+ suites.append(("tests/vector_stores/test_segmented_faiss.py", None))+ if "tests/performance/token_tracking" in targets:+ suites.append(+ ("tests/metrics/test_token_counting_dispatch_attribution.py", None)+ )+ contracts = "tests/ci/test_offline_workload_shards.py"+ if (root / contracts).is_file():+ suites.append((contracts, None))+ return suites+++def pytest_command(root, suites, profile, output, temporary=None):+ targets = [target for target, _ in suites]+ helper = "tests/performance/test_pytest_workload.py"+ if (root / helper).is_file():+ targets.append(helper)+ resources = "tests/performance/test_resources.py"+ if (root / resources).is_file():+ targets.append(resources)+ command = [+ sys.executable,+ "-m",+ "pytest",+ *targets,+ "-q",+ "-n",+ "0",+ "--tb=short",+ "--junitxml",+ str(output / "pytest.xml"),+ ]+ if temporary is not None:+ command.extend(["--basetemp", str(temporary)])+ for _, area in suites:+ if area:+ command.extend(+ [+ f"--{area}-profile",+ profile,+ f"--{area}-report",+ str(output / f"{area}.json"),+ ]+ )+ if any(+ target == "tests/performance/rag_scale_smoke" for target, _ in suites+ ):+ command.extend(["--ragscale-profile", profile])+ return command+++def require_junit(path, suites):+ cases = list(ET.parse(path).getroot().iter("testcase"))+ if not cases:+ raise ValueError("Benchmark CI collected no test cases")+ if any(+ case.find(tag) is not None+ for case in cases+ for tag in ("failure", "error", "skipped")+ ):+ raise ValueError("A benchmark assertion failed or was skipped")+ counts = {}+ for target, _ in suites:+ prefix = target.removesuffix(".py").replace("/", ".")+ count = sum(+ case.get("classname", "") == prefix+ or case.get("classname", "").startswith(prefix + ".")+ for case in cases+ )+ if count < 1:+ raise ValueError(f"No passing assertions recorded for {target}")+ counts[target] = count+ return counts+++def finite_number(value):+ return (+ isinstance(value, (int, float))+ and not isinstance(value, bool)+ and math.isfinite(value)+ and value >= 0+ )+++def require_samples(result, key="samples_ms", scale=1):+ if not isinstance(result, dict):+ raise ValueError("Invalid measurement")+ count, samples = result.get("count"), result.get(key)+ if (+ type(count) is not int+ or count < 1+ or not isinstance(samples, list)+ or len(samples) != count+ or not all(finite_number(value) for value in samples)+ ):+ raise ValueError("Missing or invalid measured samples")+ ordered = sorted(samples)+ expected = {+ "median_ms": statistics.median(samples) / scale,+ "p95_ms": ordered[math.ceil(0.95 * count) - 1] / scale,+ }+ if "max_ms" in result:+ expected["max_ms"] = ordered[-1] / scale+ for field, value in expected.items():+ actual = result.get(field)+ if not finite_number(actual) or not math.isclose(+ actual, value, rel_tol=1e-9, abs_tol=1e-12+ ):+ raise ValueError(+ f"Measurement summary disagrees with samples: {field}"+ )+++def require_report(path, area, profile):+ report = json.loads(path.read_text(encoding="utf-8"))+ if not isinstance(report, dict):+ raise ValueError(f"Invalid benchmark report: {path.name}")+ results = report.get("results")+ settings = report.get("settings")+ if (+ report.get("schema_version") != 2+ or report.get("area") != area+ or not isinstance(settings, dict)+ or settings.get("profile") != profile+ or type(settings.get("iterations")) is not int+ or settings["iterations"] < 1+ or report.get("status") != "passed"+ or not isinstance(results, list)+ or not results+ ):+ raise ValueError(f"Missing or failed benchmark results: {path.name}")+ for result in results:+ if (+ not isinstance(result, dict)+ or result.get("status") != "passed"+ or not isinstance(result.get("checks"), dict)+ or not result["checks"]+ or result.get("count") != settings["iterations"]+ ):+ raise ValueError(f"Unverified or unmeasured case in {path.name}")+ require_samples(result, "samples_ns", 1_000_000)+ require_samples(result)+ if any(+ not math.isclose(ns / 1_000_000, ms, rel_tol=1e-9, abs_tol=1e-12)+ for ns, ms in zip(+ result["samples_ns"], result["samples_ms"], strict=True+ )+ ):+ raise ValueError("Nanosecond and millisecond samples disagree")+ return len(results)+++def require_native_report(report):+ configuration = report.get("configuration")+ if (+ not isinstance(configuration, dict)+ or configuration.get("profile") != "ci"+ ):+ raise ValueError(+ "Native smoke report is missing its effective CI configuration"+ )+ if report.get("topic") == "search_quality":+ results = report.get("results")+ if (+ report.get("quality_failures") != []+ or not isinstance(results, list)+ or not results+ ):+ raise ValueError("Missing or failed retrieval-quality results")+ for result in results:+ queries = result.get("queries")+ if not isinstance(queries, list) or not queries:+ raise ValueError("Missing retrieval-quality queries")+ for query in queries:+ recall = query.get("recall_at_k")+ if (+ not finite_number(recall)+ or recall > 1+ or type(query.get("returned")) is not int+ or query["returned"] < 1+ ):+ raise ValueError("Invalid retrieval-quality query")+ return len(results)+ measurements = report.get("measurements")+ if not isinstance(measurements, dict) or not measurements:+ raise ValueError("Missing native workload measurements")+ for measurement in measurements.values():+ require_samples(measurement)+ return len(measurements)+++def collect_native_reports(temporary, suites, output, *, complete=True):+ required = {+ NATIVE_REPORTS[target][0]: NATIVE_REPORTS[target][1]+ for target, _ in suites+ if target in NATIVE_REPORTS+ }+ collected = {topic: [] for topic in required}+ candidates = set(temporary.rglob("isolated-result.json")) | set(+ temporary.rglob("http.json")+ )+ for path in sorted(candidates):+ report = json.loads(path.read_text(encoding="utf-8"))+ if not isinstance(report, dict) or report.get("topic") not in required:+ continue+ topic = report["topic"]+ destination = (+ output / "native" / f"{topic}-{len(collected[topic])}.json"+ )+ destination.parent.mkdir(parents=True, exist_ok=True)+ destination.write_text(+ json.dumps(report, indent=2) + "\n", encoding="utf-8"+ )+ count = require_native_report(report)+ collected[topic].append(+ {+ "path": str(destination),+ "source": str(path.relative_to(temporary)),+ "configuration": report["configuration"],+ "index_type": report.get("index_type"),+ "measured_cases": count,+ }+ )+ if complete:+ for topic, backends in required.items():+ if not collected[topic] or not set(backends).issubset(+ {result["index_type"] for result in collected[topic]}+ ):+ raise ValueError(+ f"Missing native workload report or backend: {topic}"+ )+ return collected+++def require_success(returncode):+ if returncode:+ raise RuntimeError(f"Benchmark pytest exited {returncode}")+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--profile", choices=("ci", "local"), default="ci")+ parser.add_argument("--output", type=Path, required=True)+ args = parser.parse_args()+ root = Path(__file__).resolve().parents[2]+ output = args.output.resolve()+ output.mkdir(parents=True, exist_ok=True)+ run_directory = Path(tempfile.mkdtemp(prefix="run-", dir=output))+ environment = dict(+ os.environ,+ LDR_PERF_PROFILE=args.profile,+ OMP_NUM_THREADS="1",+ OPENBLAS_NUM_THREADS="1",+ MKL_NUM_THREADS="1",+ NUMEXPR_NUM_THREADS="1",+ HF_HUB_OFFLINE="1",+ TRANSFORMERS_OFFLINE="1",+ TOKENIZERS_PARALLELISM="false",+ LDR_TESTING_WITH_MOCKS="false",+ PYTHONOPTIMIZE="0",+ PYTHONPATH=f"{root / 'src'}{os.pathsep}{root}",+ )+ environment["LDR_PERF_RAGSCALE_REPORT_DIR"] = str(+ run_directory / "ragscale"+ )+ summary = {+ "profile": args.profile,+ "run_directory": str(run_directory),+ "status": "running",+ }+ try:+ suites = available_suites(root)+ with tempfile.TemporaryDirectory(+ prefix=".pytest-", dir=output.parent+ ) as temporary:+ command = pytest_command(+ root, suites, args.profile, run_directory, temporary+ )+ summary.update(command=command, suites=[x[0] for x in suites])+ result = subprocess.run(+ command, cwd=root, env=environment, check=False+ )+ summary["pytest_returncode"] = result.returncode+ native = collect_native_reports(+ Path(temporary),+ suites,+ run_directory,+ complete=result.returncode == 0,+ )+ summary["native_reports"] = native+ require_success(result.returncode)+ summary["asserted_tests"] = require_junit(+ run_directory / "pytest.xml", suites+ )+ summary["measured_cases"] = {+ area: require_report(+ run_directory / f"{area}.json", area, args.profile+ )+ for _, area in suites+ if area+ }+ summary["measured_cases"].update(+ {+ topic: sum(record["measured_cases"] for record in records)+ for topic, records in native.items()+ }+ )+ summary["status"] = "passed"+ except BaseException:+ summary["status"] = "failed"+ raise+ finally:+ (output / "ci-summary.json").write_text(+ json.dumps(summary, indent=2) + "\n", encoding="utf-8"+ )+++if __name__ == "__main__":+ main()diff --git a/src/local_deep_research/chat/context.py b/src/local_deep_research/chat/context.pyindex eeb2236d51..19a67f73a7 100644--- a/src/local_deep_research/chat/context.py+++ b/src/local_deep_research/chat/context.py@@ -10,6 +10,7 @@ Multi-turn chat requires:"""from typing import Dict, Any, List, Optional+from operator import indexfrom loguru import logger@@ -362,26 +363,35 @@ class ChatContextManager:Returns combined findings text, limited in length."""- findings = []+ limit = index(self.MAX_FINDINGS_TO_INCLUDE)+ if limit > 0:+ selected = []+ for msg in reversed(self.messages):+ if msg.get("role") == "assistant" and msg.get("research_id"):+ selected.append(msg)+ if len(selected) == limit:+ break+ selected.reverse()+ else:+ # Preserve the historical [-0:] and negative-window behavior.+ selected = [+ msg+ for msg in self.messages+ if msg.get("role") == "assistant" and msg.get("research_id")+ ][-limit:]- for msg in self.messages:- if msg.get("role") == "assistant" and msg.get("research_id"):- # Strip BEFORE truncating: truncation mid-citation would- # cut a marker into a shape no pattern can match (residue- # like "[](htt..." would survive into follow-up context).- content = self._strip_citation_markers(msg.get("content") or "")- # Summarize long responses - take first part- if len(content) > 500:- # Try to find a natural break point- break_point = content.find("\n\n", 300)- if break_point == -1 or break_point > 600:- break_point = 500- content = content[:break_point] + "..."- findings.append(content)-- # Keep only recent findings- recent_findings = findings[-self.MAX_FINDINGS_TO_INCLUDE :]- return "\n\n---\n\n".join(recent_findings)+ findings = []+ for msg in selected:+ # Strip BEFORE truncating, but only for findings that survive+ # the window. Discarded history needs neither parsing nor copies.+ content = self._strip_citation_markers(msg.get("content") or "")+ if len(content) > 500:+ break_point = content.find("\n\n", 300)+ if break_point == -1 or break_point > 600:+ break_point = 500+ content = content[:break_point] + "..."+ findings.append(content)+ return "\n\n---\n\n".join(findings)def _get_key_entities(self) -> List[str]:"""Get key entities from accumulated context."""diff --git a/tests/ci/standalone/test_performance_pytest.py b/tests/ci/standalone/test_performance_pytest.pynew file mode 100644index 0000000000..d7993f3c71--- /dev/null+++ b/tests/ci/standalone/test_performance_pytest.py@@ -0,0 +1,349 @@+"""Reject absent, skipped and failed benchmark evidence without app imports."""++import importlib.util+from copy import deepcopy+import json+from pathlib import Path+import tempfile+import unittest+from unittest.mock import patch++SCRIPT = (+ Path(__file__).resolve().parents[3] / "scripts/ci/performance_pytest.py"+)+SPEC = importlib.util.spec_from_file_location("performance_ci", SCRIPT)+CI = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(CI)+++class BenchmarkEvidenceTests(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name)++ def good_report(self):+ return {+ "schema_version": 2,+ "area": "chatctx",+ "settings": {"profile": "ci", "iterations": 2},+ "status": "passed",+ "results": [+ {+ "status": "passed",+ "checks": {"rows": 8},+ "count": 2,+ "samples_ns": [1_000_000, 2_000_000],+ "samples_ms": [1, 2],+ "median_ms": 1.5,+ "p95_ms": 2,+ }+ ],+ }++ def native_report(self, topic="database_performance", backend=None):+ return {+ "topic": topic,+ "index_type": backend,+ "configuration": {"profile": "ci", "iterations": 2, "rows": 8},+ "measurements": {+ "read": {+ "count": 2,+ "samples_ms": [1, 2],+ "median_ms": 1.5,+ "p95_ms": 2,+ "max_ms": 2,+ }+ },+ }++ def mini_test(self, relative, source):+ target = self.root / relative+ target.parent.mkdir(parents=True, exist_ok=True)+ directory = target.parent+ while directory != self.root:+ (directory / "__init__.py").write_text("", encoding="utf-8")+ directory = directory.parent+ target.write_text(source, encoding="utf-8")+ return target++ def run_mini(self, output):+ with (+ patch.object(+ CI,+ "__file__",+ str(self.root / "scripts/ci/performance_pytest.py"),+ ),+ patch.object(+ CI.sys, "argv", ["performance_pytest", "--output", str(output)]+ ),+ ):+ CI.main()++ def xml(self, body):+ path = self.root / "results.xml"+ path.write_text(+ f"<testsuites><testsuite>{body}</testsuite></testsuites>",+ encoding="utf-8",+ )+ return path++ def test_empty_checkout_is_not_a_successful_benchmark_run(self):+ with self.assertRaises(ValueError):+ CI.available_suites(self.root)++ def test_cache_directories_do_not_create_phantom_suites_or_core_targets(+ self,+ ):+ self.mini_test(+ "tests/performance/database/test_connection_workload.py", ""+ )+ for target in (+ "content_extraction",+ "token_tracking",+ "rag_scale_smoke",+ ):+ (self.root / "tests/performance" / target / "__pycache__").mkdir(+ parents=True+ )+ self.assertEqual(+ CI.available_suites(self.root),+ [+ (+ "tests/performance/database/test_connection_workload.py",+ None,+ ),+ ],+ )++ def test_each_selected_suite_must_record_passing_assertions(self):+ path = self.xml(+ '<testcase classname="tests.performance.chat_context.test_offline" name="check"/>'+ )+ suites = [("tests/performance/chat_context", "chatctx")]+ self.assertEqual(CI.require_junit(path, suites), {suites[0][0]: 1})+ with self.assertRaises(ValueError):+ CI.require_junit(+ path, suites + [("tests/performance/token_tracking", "tokens")]+ )++ def test_no_collection_skips_and_failures_are_rejected(self):+ suites = [("tests/performance/chat_context", "chatctx")]+ for body in [+ "",+ *[+ '<testcase classname="tests.performance.chat_context.test_offline">'+ f"<{tag}/></testcase>"+ for tag in ["skipped", "failure", "error"]+ ],+ ]:+ with self.subTest(body=body), self.assertRaises(ValueError):+ CI.require_junit(self.xml(body), suites)++ def test_failed_or_unverified_measurements_are_rejected(self):+ path = self.root / "report.json"+ good = self.good_report()+ path.write_text(json.dumps(good), encoding="utf-8")+ self.assertEqual(CI.require_report(path, "chatctx", "ci"), 1)+ bad = []+ for key, value in [+ ("status", "failed"),+ ("results", []),+ ("area", "tokens"),+ ("schema_version", 1),+ ]:+ report = deepcopy(good)+ report[key] = value+ bad.append(report)+ report = deepcopy(good)+ report["settings"]["profile"] = "local"+ bad.append(report)+ for key, value in [+ ("checks", {}),+ ("count", 0),+ ("count", True),+ ("samples_ns", []),+ ("samples_ms", [1, float("nan")]),+ ("samples_ns", [1_000_000, float("inf")]),+ ("p95_ms", 1),+ ("median_ms", float("nan")),+ ]:+ report = deepcopy(good)+ report["results"][0][key] = value+ bad.append(report)+ for report in bad:+ path.write_text(json.dumps(report), encoding="utf-8")+ with self.subTest(report=report), self.assertRaises(ValueError):+ CI.require_report(path, "chatctx", "ci")++ def test_native_measurements_are_retained_and_require_both_backends(self):+ temporary = self.root / "pytest"+ temporary.mkdir()+ output = self.root / "artifacts"+ suites = [("tests/performance/rag/test_storage_workload.py", None)]+ first = temporary / "flat" / "isolated-result.json"+ first.parent.mkdir()+ first.write_text(+ json.dumps(self.native_report("rag_performance", "flat"))+ )+ with self.assertRaisesRegex(ValueError, "backend"):+ CI.collect_native_reports(temporary, suites, output)+ second = temporary / "hnsw" / "isolated-result.json"+ second.parent.mkdir()+ second.write_text(+ json.dumps(self.native_report("rag_performance", "hnsw"))+ )+ result = CI.collect_native_reports(temporary, suites, output)+ self.assertEqual(+ {r["index_type"] for r in result["rag_performance"]},+ {"flat", "hnsw"},+ )+ first.unlink()+ second.unlink()+ for record in result["rag_performance"]:+ report = json.loads(Path(record["path"]).read_text())+ self.assertEqual(+ report["measurements"]["read"]["samples_ms"], [1, 2]+ )+ self.assertEqual(record["configuration"]["rows"], 8)++ def test_missing_or_invalid_native_measurements_are_rejected(self):+ with self.assertRaisesRegex(ValueError, "Missing native"):+ CI.collect_native_reports(+ self.root,+ [+ (+ "tests/performance/database/test_connection_workload.py",+ None,+ )+ ],+ self.root / "output",+ )+ for key, value in [("configuration", {}), ("measurements", {})]:+ report = self.native_report()+ report[key] = value+ with self.subTest(key=key), self.assertRaises(ValueError):+ CI.require_native_report(report)++ def test_quality_reports_require_successful_nonempty_queries(self):+ good = {+ "topic": "search_quality",+ "configuration": {"profile": "ci"},+ "quality_failures": [],+ "results": [{"queries": [{"recall_at_k": 1.0, "returned": 3}]}],+ }+ self.assertEqual(CI.require_native_report(good), 1)+ for report in [+ dict(good, quality_failures=["cosine/reopen"]),+ dict(good, results=[]),+ dict(good, results=[{"queries": []}]),+ dict(+ good,+ results=[+ {"queries": [{"recall_at_k": float("nan"), "returned": 3}]}+ ],+ ),+ ]:+ with self.subTest(report=report), self.assertRaises(ValueError):+ CI.require_native_report(report)++ def test_existing_manual_workflow_contracts_are_also_selected(self):+ target = (+ self.root / "tests/performance/database/test_connection_workload.py"+ )+ target.parent.mkdir(parents=True)+ target.touch()+ contracts = self.root / "tests/ci/test_offline_workload_shards.py"+ contracts.parent.mkdir()+ contracts.touch()+ self.assertIn(+ ("tests/ci/test_offline_workload_shards.py", None),+ CI.available_suites(self.root),+ )++ def test_ci_uses_one_worker_and_explicit_profile_per_present_area(self):+ command = CI.pytest_command(+ self.root,+ [("tests/performance/chat_context", "chatctx")],+ "ci",+ self.root,+ )+ self.assertEqual(command[command.index("-n") + 1], "0")+ self.assertEqual(command[command.index("--chatctx-profile") + 1], "ci")+ self.assertIn("--chatctx-report", command)++ def test_real_child_does_not_silently_skip_integration_assertions(self):+ target = self.mini_test(+ "tests/performance/database/test_connection_workload.py",+ "import json\nimport os\nimport pytest\n"+ "@pytest.fixture(autouse=True)\n"+ "def real_mode():\n"+ ' if os.environ.get("LDR_TESTING_WITH_MOCKS") != "false":\n'+ ' pytest.skip("integration disabled")\n'+ "def test_required_assertion(tmp_path):\n"+ " assert __debug__\n"+ ' assert os.environ["LDR_PERF_PROFILE"] == "ci"\n'+ ' assert os.environ["OMP_NUM_THREADS"] == "1"\n',+ )+ target.write_text(+ target.read_text()+ + ' (tmp_path / "isolated-result.json").write_text('+ + repr(json.dumps(self.native_report()))+ + ")\n"+ )+ output = self.root / "results"+ self.run_mini(output)+ result = json.loads(+ (output / "ci-summary.json").read_text(encoding="utf-8")+ )+ self.assertEqual(result["status"], "passed")+ self.assertEqual(+ result["asserted_tests"][str(target.relative_to(self.root))], 1+ )+ native = result["native_reports"]["database_performance"]+ self.assertEqual(len(native), 1)+ self.assertTrue(Path(native[0]["path"]).is_file())+ self.assertFalse(+ (+ output.parent+ / Path(+ result["command"][result["command"].index("--basetemp") + 1]+ )+ ).exists()+ )++ def test_reused_output_cannot_substitute_previous_measurements(self):+ target = self.mini_test(+ "tests/performance/chat_context/test_measurement.py",+ "import os\nfrom pathlib import Path\n"+ "def test_measurement(request):\n"+ " assert 2 + 2 == 4\n"+ ' if os.environ.get("WRITE_BENCHMARK_REPORT") == "yes":\n'+ ' Path(request.config.getoption("--chatctx-report")).write_text('+ + repr(json.dumps(self.good_report()))+ + ")\n",+ )+ (target.parent / "conftest.py").write_text(+ "def pytest_addoption(parser):\n"+ ' parser.addoption("--chatctx-profile")\n'+ ' parser.addoption("--chatctx-report")\n'+ )+ output = self.root / "results"+ with patch.dict("os.environ", {"WRITE_BENCHMARK_REPORT": "yes"}):+ self.run_mini(output)+ first = json.loads((output / "ci-summary.json").read_text())+ with (+ patch.dict("os.environ", {"WRITE_BENCHMARK_REPORT": "no"}),+ self.assertRaises(FileNotFoundError),+ ):+ self.run_mini(output)+ second = json.loads((output / "ci-summary.json").read_text())+ self.assertEqual(second["status"], "failed")+ self.assertNotEqual(first["run_directory"], second["run_directory"])+ self.assertTrue(+ (Path(first["run_directory"]) / "chatctx.json").exists()+ )+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/performance/_shared/pytest_workload.py b/tests/performance/_shared/pytest_workload.pynew file mode 100644index 0000000000..7c7defb5e7--- /dev/null+++ b/tests/performance/_shared/pytest_workload.py@@ -0,0 +1,294 @@+"""Small pytest workloads with explicit profiles and optional timing budgets."""++from dataclasses import dataclass+import hashlib+from importlib.metadata import PackageNotFoundError, version+import json+import math+import os+from pathlib import Path+import platform+import statistics+import subprocess+import sys+import time++import pytest+++def add_options(parser, area):+ group = parser.getgroup(f"{area} performance")+ for field in (+ "profile",+ "size",+ "iterations",+ "warmups",+ "report",+ "max-p95-ms",+ ):+ group.addoption(f"--{area}-{field}", default=None)+++def resolve(config, area, profiles):+ def value(field, default):+ cli = config.getoption(f"--{area}-{field}")+ key = field.replace("-", "_").upper()+ if cli is not None:+ return cli+ environment = os.environ.get(f"LDR_PERF_{area.upper()}_{key}")+ return environment if environment not in (None, "") else default++ profile = value("profile", os.environ.get("LDR_PERF_PROFILE") or "ci")+ if profile not in profiles:+ raise pytest.UsageError("Performance profile must be ci or local")+ result = {"profile": profile}+ for field in ("size", "iterations", "warmups"):+ try:+ number = int(value(field, profiles[profile][field]))+ except (TypeError, ValueError):+ raise pytest.UsageError(+ f"{area} {field} must be an integer"+ ) from None+ if number < (0 if field == "warmups" else 1):+ raise pytest.UsageError(f"Invalid {area} {field}")+ result[field] = number+ result["report"] = value("report", None)+ limit = value("max-p95-ms", None)+ if limit is not None:+ try:+ limit = float(limit)+ except (TypeError, ValueError):+ raise pytest.UsageError("Timing budget must be numeric") from None+ if not math.isfinite(limit) or limit <= 0:+ raise pytest.UsageError("Timing budget must be positive and finite")+ result["max_p95_ms"] = limit+ return result+++@dataclass+class Workload:+ area: str+ settings: dict++ def __post_init__(self):+ self.results = []++ def measure(+ self,+ name,+ size,+ prepare,+ operation,+ verify,+ *,+ units=1,+ unit="batch",+ parameters=None,+ ):+ """Time only the operation, excluding setup, checks and prior outputs."""+ if not isinstance(units, int) or isinstance(units, bool) or units < 1:+ raise ValueError("Work units must be a positive integer")+ samples = []+ result = dict(+ case=name,+ size=size,+ unit=unit,+ units=units,+ parameters=parameters or {},+ samples_ns=samples,+ samples_ms=[],+ status="running",+ )+ self.results.append(result)+ phase = "prepare"+ try:+ total = self.settings["warmups"] + self.settings["iterations"]+ for number in range(total):+ state = output = None+ try:+ phase = "prepare"+ state = prepare()+ phase = "operation"+ begun = time.perf_counter_ns()+ output = operation(state)+ elapsed = time.perf_counter_ns() - begun+ if elapsed < 0:+ raise ValueError("Negative benchmark duration")+ if number >= self.settings["warmups"]:+ samples.append(elapsed)+ result["samples_ms"].append(elapsed / 1_000_000)+ phase = "verify"+ result["checks"] = verify(state, output)+ finally:+ # Replacing last iteration's result inside the next+ # timed assignment charged its destruction to that call.+ output = state = None+ except BaseException as error:+ result.update(+ status="failed",+ error_type=type(error).__name__,+ failure_phase=phase,+ )+ raise+ ordered = sorted(samples)+ median = statistics.median(samples)+ p95 = ordered[math.ceil(0.95 * len(ordered)) - 1]+ result.update(+ status="passed",+ count=len(samples),+ median_ms=median / 1_000_000,+ p95_ms=p95 / 1_000_000,+ median_ns_per_unit=median / units,+ p95_ns_per_unit=p95 / units,+ units_per_second=units * 1_000_000_000 / median if median else None,+ )++ def finish(self, packages=(), *, modules=()):+ root = Path(__file__).resolve().parents[3]++ def git(*arguments, cwd=root):+ try:+ completed = subprocess.run(+ ["git", *arguments],+ cwd=cwd,+ capture_output=True,+ text=True,+ )+ except OSError:+ return None+ return (+ completed.stdout.strip() if completed.returncode == 0 else None+ )++ revision, dirty = git("rev-parse", "HEAD"), git("status", "--porcelain")+ loaded_sources = {}+ for name in modules:+ filename = getattr(sys.modules.get(name), "__file__", None)+ path = Path(filename).resolve() if filename else None+ tracked = (+ path is not None+ and git(+ "ls-files",+ "--error-unmatch",+ "--",+ str(path),+ cwd=path.parent,+ )+ is not None+ )+ try:+ digest = (+ hashlib.sha256(path.read_bytes()).hexdigest()+ if path+ else None+ )+ except OSError:+ digest = None+ source_dirty = (+ git("status", "--porcelain", cwd=path.parent)+ if tracked+ else None+ )+ loaded_sources[name] = dict(+ path=str(path) if path else None,+ revision=git("rev-parse", "HEAD", cwd=path.parent)+ if tracked+ else None,+ dirty=bool(source_dirty) if source_dirty is not None else None,+ sha256=digest,+ )+ source_revisions = {+ value["revision"] for value in loaded_sources.values()+ }+ versions = {}+ for name in packages:+ try:+ versions[name] = version(name)+ except PackageNotFoundError:+ versions[name] = None+ budget = self.settings["max_p95_ms"]+ failures = [+ f"{result['case']}[{result['size']}]"+ for result in self.results+ if budget is not None and result.get("p95_ms", 0) > budget+ ]+ curves = {}+ for result in self.results:+ if result["status"] == "passed":+ curves.setdefault((result["case"], result["unit"]), []).append(+ result+ )+ scaling = []+ for (name, unit), points in curves.items():+ points.sort(key=lambda point: point["size"])+ for smaller, larger in zip(points, points[1:]):+ if (+ larger["size"] > smaller["size"]+ and smaller["median_ms"] > 0+ ):+ latency = larger["median_ms"] / smaller["median_ms"]+ work = larger["units"] / smaller["units"]+ scaling.append(+ dict(+ case=name,+ unit=unit,+ sizes=[smaller["size"], larger["size"]],+ latency_ratio=latency,+ work_ratio=work,+ normalized_latency_ratio=latency / work,+ )+ )+ passed = bool(self.results) and all(+ result["status"] == "passed" for result in self.results+ )+ report = dict(+ schema_version=2,+ area=self.area,+ status="passed" if passed and not failures else "failed",+ settings=self.settings,+ results=self.results,+ observed_scaling=scaling,+ budget_failures=failures,+ environment=dict(+ python=platform.python_version(),+ platform=platform.platform(),+ processor=platform.processor(),+ cpu_count=os.cpu_count(),+ native_threads={+ name: os.environ.get(name)+ for name in (+ "OMP_NUM_THREADS",+ "OPENBLAS_NUM_THREADS",+ "MKL_NUM_THREADS",+ )+ },+ pytest_worker=os.environ.get("PYTEST_XDIST_WORKER"),+ timing_scope="operation in pytest process; preparation, verification and result release excluded",+ rate_scope="batch duration divided by declared work; per-unit p95 is not an individual-request quantile",+ harness_revision=revision,+ harness_dirty=bool(dirty) if dirty is not None else None,+ source_revision=next(iter(source_revisions))+ if len(source_revisions) == 1+ else None,+ source_dirty=any(+ value["dirty"] for value in loaded_sources.values()+ )+ if loaded_sources+ and all(+ value["dirty"] is not None+ for value in loaded_sources.values()+ )+ else None,+ loaded_sources=loaded_sources,+ packages=versions,+ ),+ )+ if self.settings["report"]:+ path = Path(self.settings["report"])+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(+ json.dumps(report, indent=2) + "\n", encoding="utf-8"+ )+ if failures:+ pytest.fail("Explicit p95 budget exceeded: " + ", ".join(failures))+ return reportdiff --git a/tests/performance/chat_context/README.md b/tests/performance/chat_context/README.mdnew file mode 100644index 0000000000..78bd547842--- /dev/null+++ b/tests/performance/chat_context/README.md@@ -0,0 +1,22 @@+These offline tests measure actual raw chat-context construction while varying+history length, long/short/cited responses and interleaved progress steps. The+complete expected context is generated independently. Assertions gate latest+findings, order, user text, progress filtering, fresh content and a linear+stripping-work budget. Window limits retain their existing zero/negative behavior.++```bash+pdm run pytest tests/performance/chat_context -q \+ --chatctx-profile local --chatctx-report /tmp/chat-context.json+```++CI uses 8/16 findings, two iterations after one warmup; local uses 250/500,+seven after two. `--chatctx-{profile,size,iterations,warmups,report,max-p95-ms}`+overrides `LDR_PERF_CHATCTX_*`, with shared `LDR_PERF_PROFILE` fallback.+Constructor/input preparation, assertions, cleanup and log-sink I/O are untimed.+The operation includes a counter around real citation stripping. There are no+LLM or network calls. Per-unit p95 is a scaled batch statistic, not item latency.++The automatic Performance Assertions workflow runs serial CI profiles, checks+JUnit for actual passing tests in every selected suite and rejects missing,+failed or skipped evidence. Reports retain raw timings and implementation hashes.+Wall-clock gates are opt-in; correctness and work limits always gate CI.diff --git a/tests/performance/chat_context/__init__.py b/tests/performance/chat_context/__init__.pynew file mode 100644index 0000000000..1c0e666339--- /dev/null+++ b/tests/performance/chat_context/__init__.py@@ -0,0 +1 @@+"""Offline performance contracts for raw chat context."""diff --git a/tests/performance/chat_context/conftest.py b/tests/performance/chat_context/conftest.pynew file mode 100644index 0000000000..2c36080315--- /dev/null+++ b/tests/performance/chat_context/conftest.py@@ -0,0 +1,38 @@+from types import SimpleNamespace++import pytest++from local_deep_research.chat import context+from tests.performance._shared.pytest_workload import (+ Workload,+ add_options,+ resolve,+)++PROFILES = {+ "ci": dict(size=8, iterations=2, warmups=1),+ "local": dict(size=250, iterations=7, warmups=2),+}+++def pytest_addoption(parser):+ add_options(parser, "chatctx")++[email protected](autouse=True)+def omit_log_sink_io(monkeypatch):+ monkeypatch.setattr(+ context, "logger", SimpleNamespace(info=lambda *a, **k: None)+ )++[email protected](scope="session")+def chat_workload(request):+ workload = Workload("chatctx", resolve(request.config, "chatctx", PROFILES))+ yield workload+ workload.finish(+ modules=(+ "local_deep_research.chat.context",+ "local_deep_research.chat.citation_stripping",+ )+ )diff --git a/tests/performance/chat_context/test_offline.py b/tests/performance/chat_context/test_offline.pynew file mode 100644index 0000000000..14f67090c9--- /dev/null+++ b/tests/performance/chat_context/test_offline.py@@ -0,0 +1,131 @@+"""Gate output and linear work; measure the real raw-context construction."""++import pytest++from local_deep_research.chat.context import ChatContextManager+++def history(count, kind, steps=False):+ messages, findings = [], []+ for number in range(count):+ prefix = f"Finding {number}: "+ if kind == "short":+ body = prefix + "Evidence. " * 8+ expected = body+ else:+ marker = (+ "[[7]](https://example.test/paper)"+ if kind == "citations"+ else ""+ )+ body = prefix + marker + "x" * 450 + "\n\n" + "Support. " * 60+ expected = prefix + "x" * 450 + "..."+ messages.extend(+ [+ dict(role="user", content=f"User [9] {number}"),+ dict(+ role="assistant",+ research_id=f"research-{number}",+ content=body,+ ),+ ]+ )+ if steps:+ messages.append(+ dict(+ message_type="step",+ role="assistant",+ research_id="progress",+ content="Excluded progress",+ )+ )+ findings.append(expected)+ return messages, findings++[email protected]("kind", ["short", "long", "citations"])[email protected]("steps", [False, True], ids=["turns", "with_steps"])+def test_history_growth_preserves_recent_findings_and_bounds_work(+ chat_workload, kind, steps+):+ for size in (+ chat_workload.settings["size"],+ chat_workload.settings["size"] * 2,+ ):+ messages, findings = history(size, kind, steps)+ expected = "\n\n---\n\n".join(findings[-5:])++ def prepare():+ manager = ChatContextManager("performance-session", messages)+ manager.MAX_FINDINGS_TO_INCLUDE = 5+ actual = manager._strip_citation_markers+ calls = []++ def counted(text):+ calls.append(len(text))+ return actual(text)++ manager._strip_citation_markers = counted+ return manager, calls++ def verify(state, output):+ manager, calls = state+ try:+ assert output == {+ "session_id": "performance-session",+ "original_query": "User [9] 0",+ "accumulated_findings": expected,+ "past_findings": expected,+ "key_entities": [],+ "topics": [],+ "is_multi_turn": True,+ "turn_count": size * 2,+ }+ # Allows future window-first processing while rejecting repeated+ # processing beyond one pass through qualifying messages.+ assert len(calls) <= size+ return dict(+ history_findings=size,+ retained_findings=min(5, size),+ stripping_calls=len(calls),+ stripped_input_chars=sum(calls),+ output_chars=len(expected),+ )+ finally:+ del manager._strip_citation_markers++ chat_workload.measure(+ f"raw_{kind}_{'steps' if steps else 'turns'}",+ size,+ prepare,+ lambda state: state[0].build_research_context(),+ verify,+ units=size,+ unit="history_finding",+ parameters=dict(+ kind=kind,+ retained_findings=5,+ progress_steps=steps,+ instrumented_work_count=True,+ constructor_timed=False,+ ),+ )++[email protected]("limit", [0, -1, 1, 5, 20])+def test_findings_window_boundaries_keep_existing_selection_semantics(limit):+ messages, expected = history(8, "long", steps=True)+ manager = ChatContextManager("boundaries", messages)+ manager.MAX_FINDINGS_TO_INCLUDE = limit+ assert manager._extract_findings_from_history() == "\n\n---\n\n".join(+ expected[-limit:]+ )+++def test_changed_latest_response_is_observed_on_next_build():+ messages, _ = history(8, "short")+ manager = ChatContextManager("freshness", messages)+ first = manager.build_research_context()["past_findings"]+ manager.messages[-1]["content"] = "Updated latest finding"+ second = manager.build_research_context()["past_findings"]+ assert first != second and second.endswith("Updated latest finding")diff --git a/tests/performance/chat_context/test_retained_work.py b/tests/performance/chat_context/test_retained_work.pynew file mode 100644index 0000000000..0b71af3e81--- /dev/null+++ b/tests/performance/chat_context/test_retained_work.py@@ -0,0 +1,82 @@+"""Raw context should spend parsing work only on retained research findings."""++import pytest++from local_deep_research.chat.context import ChatContextManager++[email protected]("limit", [-20, -1, 0, 1, 5, 20, True, False])+def test_selected_findings_keep_slice_semantics_and_only_strip_retained_text(+ monkeypatch, limit+):+ messages = []+ for number in range(8):+ messages.extend(+ [+ {"role": "user", "content": f"Question [9] {number}"},+ {+ "role": "assistant",+ "research_id": str(number),+ "content": f"Finding {number} [1]",+ },+ {+ "role": "assistant",+ "message_type": "step",+ "research_id": "step",+ "content": "Progress",+ },+ {"role": "assistant", "content": "No research attached"},+ ]+ )+ manager = ChatContextManager("window-work", messages)+ manager.MAX_FINDINGS_TO_INCLUDE = limit+ expected_numbers = list(range(8))[-limit:]+ calls = []+ original = ChatContextManager._strip_citation_markers++ def strip(self, text):+ calls.append(text)+ return original(self, text)++ monkeypatch.setattr(ChatContextManager, "_strip_citation_markers", strip)+ assert manager._extract_findings_from_history() == "\n\n---\n\n".join(+ f"Finding {number}" for number in expected_numbers+ )+ assert calls == [f"Finding {number} [1]" for number in expected_numbers]+ assert messages[0]["content"] == "Question [9] 0"+++def test_positive_window_does_not_walk_the_discarded_history():+ class Message(dict):+ role_reads = 0++ def get(self, key, *args):+ if key == "role":+ Message.role_reads += 1+ return super().get(key, *args)++ messages = []+ for number in range(500):+ messages.extend(+ [+ Message(role="user", content="Question"),+ Message(+ role="assistant",+ research_id=str(number),+ content=f"Finding {number}",+ ),+ ]+ )+ manager = ChatContextManager("bounded-window", messages)+ Message.role_reads = 0+ assert manager._extract_findings_from_history().split("\n\n---\n\n") == [+ f"Finding {number}" for number in range(495, 500)+ ]+ assert Message.role_reads <= 10+++def test_non_integer_findings_window_is_rejected():+ manager = ChatContextManager("invalid-window", [])+ manager.MAX_FINDINGS_TO_INCLUDE = 1.5+ with pytest.raises(TypeError):+ manager._extract_findings_from_history()diff --git a/tests/performance/citation_metadata/README.md b/tests/performance/citation_metadata/README.mdnew file mode 100644index 0000000000..9ccdf35567--- /dev/null+++ b/tests/performance/citation_metadata/README.md@@ -0,0 +1,26 @@+# Citation metadata normalization++These offline workloads measure the actual `normalize_citation()` function across+web/non-academic, OpenAlex string/structured authors, arXiv and PubMed inputs.+They vary source count and, separately, authors per source at a fixed batch size.++```bash+pdm run pytest tests/performance/citation_metadata -n 0 -q \+ --citenorm-profile local --citenorm-report /tmp/citation-metadata.json+```++CI uses sizes 8/16 and two measured iterations after one warmup; local uses+128/256 and seven after two. CLI `--citenorm-{profile,size,iterations,warmups,+report,max-p95-ms}` overrides `LDR_PERF_CITENORM_*`; `LDR_PERF_PROFILE` supplies+the shared profile fallback. Timing ceilings are opt-in.++The complete independent expected metadata includes authors/CSL, dates, DOI and+engine-specific identifiers. Inputs must remain unchanged and outputs must be+JSON-serializable with the expected types. Negative controls erase authors or+alter dates/DOIs. Repeated calls must observe changed input without changing a+previous result. Author/name parsing work must remain linear. Instrumentation+adds counter-call overhead; input generation/copying, validation, serialization+checks and cleanup are outside the timed normalization batch. No model, database+or network calls are made. Per-unit p95 is a scaled batch statistic, not a+single-author latency quantile. The automatic CI job retains raw samples and+requires passing assertions and fresh reports.diff --git a/tests/performance/citation_metadata/__init__.py b/tests/performance/citation_metadata/__init__.pynew file mode 100644index 0000000000..e5e3b79414--- /dev/null+++ b/tests/performance/citation_metadata/__init__.py@@ -0,0 +1 @@+"""Offline academic citation metadata normalization workloads."""diff --git a/tests/performance/citation_metadata/conftest.py b/tests/performance/citation_metadata/conftest.pynew file mode 100644index 0000000000..30d3f5e3fc--- /dev/null+++ b/tests/performance/citation_metadata/conftest.py@@ -0,0 +1,27 @@+import pytest++from tests.performance._shared.pytest_workload import (+ Workload,+ add_options,+ resolve,+)++PROFILES = {+ "ci": dict(size=8, iterations=2, warmups=1),+ "local": dict(size=128, iterations=7, warmups=2),+}+++def pytest_addoption(parser):+ add_options(parser, "citenorm")++[email protected](scope="session")+def citation_workload(request):+ workload = Workload(+ "citenorm", resolve(request.config, "citenorm", PROFILES)+ )+ yield workload+ workload.finish(+ modules=("local_deep_research.utilities.citation_normalizer",)+ )diff --git a/tests/performance/citation_metadata/corpus.py b/tests/performance/citation_metadata/corpus.pynew file mode 100644index 0000000000..7464e5c860--- /dev/null+++ b/tests/performance/citation_metadata/corpus.py@@ -0,0 +1,95 @@+"""Independent expected CSL data for a fixed synthetic citation vocabulary."""++from copy import deepcopy+import json+++def corpus(count, kind, authors_per_source=4):+ sources, expected = [], []+ for number in range(count):+ title = f"Synthetic paper {number}"+ if kind == "web":+ sources.append(+ dict(+ title=title,+ source_engine="web",+ url=f"https://example.test/page/{number}",+ )+ )+ expected.append(None)+ continue+ engine = "openalex" if kind.startswith("openalex") else kind+ doi = f"10.1234/measurement.{number}"+ names = [+ {"given": f"Given{a}", "family": f"Family{a}"}+ for a in range(authors_per_source)+ ]+ source = dict(+ title=title,+ source_engine=" " + engine.upper() + " ",+ authors=[f"Given{a} Family{a}" for a in range(authors_per_source)],+ doi="https://doi.org/" + doi,+ publication_date="2024-02-03",+ journal="Measurement Journal",+ volume="12",+ issue="3",+ pages="1-9",+ url=f"https://example.test/paper/{number}",+ )+ if kind == "openalex_csl":+ names = [dict(name, suffix="Jr.") for name in names]+ source["metadata"] = {+ "authors_csl": [+ dict(name, affiliation={"name": "Unused affiliation"})+ for name in names+ ]+ }+ source["authors"] = ["Ignored Display Name"]+ source.pop("doi")+ source["externalIds"] = {"DOI": "doi:" + doi}+ if engine == "arxiv":+ source.update(+ arxiv_id=f"2401.{number:05d}", journal_ref="Measurement Journal"+ )+ if engine == "pubmed":+ source["pmid"] = str(30000000 + number)+ csl = {+ "type": "article-journal",+ "title": title,+ "author": deepcopy(names),+ "container-title": "Measurement Journal",+ "DOI": doi,+ "volume": "12",+ "issue": "3",+ "page": "1-9",+ "issued": {"date-parts": [[2024, 2, 3]]},+ "URL": source["url"],+ }+ normalized = dict(+ source_engine=engine,+ doi=doi,+ authors=deepcopy(names),+ publication_date="2024-02-03",+ year=2024,+ volume="12",+ issue="3",+ pages="1-9",+ container_title="Measurement Journal",+ item_type="article-journal",+ csl_json=csl,+ )+ if engine == "arxiv":+ normalized["arxiv_id"] = source["arxiv_id"]+ if engine == "pubmed":+ normalized["pmid"] = source["pmid"]+ sources.append(source)+ expected.append(normalized)+ return sources, expected+++def verify_batch(inputs, output, original, expected):+ assert output == expected, "Citation output mismatch"+ assert inputs == original, "Citation input was mutated"+ assert json.dumps(output, sort_keys=True, allow_nan=False) == json.dumps(+ expected, sort_keys=True, allow_nan=False+ ), "Citation JSON type mismatch"diff --git a/tests/performance/citation_metadata/test_offline.py b/tests/performance/citation_metadata/test_offline.pynew file mode 100644index 0000000000..85bf671639--- /dev/null+++ b/tests/performance/citation_metadata/test_offline.py@@ -0,0 +1,137 @@+"""Measure actual normalization and require complete independent CSL results."""++from copy import deepcopy++import pytest++from local_deep_research.utilities import citation_normalizer as normalizer++from .corpus import corpus, verify_batch+++def measure(+ citation_workload, monkeypatch, name, size, sources, expected, authors+):+ calls = {"authors": 0, "names": 0}+ parse_authors, parse_name = (+ normalizer._parse_authors_list,+ normalizer._parse_name,+ )++ def counted_authors(*args):+ calls["authors"] += 1+ return parse_authors(*args)++ def counted_name(*args):+ calls["names"] += 1+ return parse_name(*args)++ with monkeypatch.context() as patch:+ patch.setattr(normalizer, "_parse_authors_list", counted_authors)+ patch.setattr(normalizer, "_parse_name", counted_name)++ def prepare():+ calls.update(authors=0, names=0)+ return deepcopy(sources)++ def verify(state, output):+ verify_batch(state, output, sources, expected)+ academic = sum(item is not None for item in expected)+ assert calls["authors"] <= academic+ assert calls["names"] <= academic * authors+ return dict(+ sources=len(sources),+ academic=academic,+ authors_per_source=authors,+ author_parses=calls["authors"],+ name_parses=calls["names"],+ )++ citation_workload.measure(+ name,+ size,+ prepare,+ lambda state: [+ normalizer.normalize_citation(source) for source in state+ ],+ verify,+ units=len(sources) * max(authors, 1),+ unit="source_author",+ parameters=dict(+ sources=len(sources),+ authors_per_source=authors,+ instrumentation=True,+ input_copy_timed=False,+ ),+ )+++ "kind", ["web", "openalex_strings", "openalex_csl", "arxiv", "pubmed"]+)+def test_source_growth_preserves_complete_metadata(+ citation_workload, monkeypatch, kind+):+ for size in (+ citation_workload.settings["size"],+ citation_workload.settings["size"] * 2,+ ):+ sources, expected = corpus(size, kind)+ measure(+ citation_workload,+ monkeypatch,+ "sources_" + kind,+ size,+ sources,+ expected,+ 4 if kind != "web" else 0,+ )++[email protected]("kind", ["openalex_strings", "openalex_csl"])+def test_author_growth_at_fixed_source_count(+ citation_workload, monkeypatch, kind+):+ for size in (+ citation_workload.settings["size"],+ citation_workload.settings["size"] * 2,+ ):+ sources, expected = corpus(4, kind, size)+ measure(+ citation_workload,+ monkeypatch,+ "authors_" + kind,+ size,+ sources,+ expected,+ size,+ )++[email protected]("fault", ["authors", "date", "doi"])+def test_full_output_oracle_rejects_lost_citation_fields(fault):+ sources, expected = corpus(2, "openalex_csl")+ inputs = deepcopy(sources)+ output = [normalizer.normalize_citation(source) for source in inputs]+ verify_batch(inputs, output, sources, expected)+ if fault == "authors":+ output[0]["authors"] = []+ elif fault == "date":+ output[0]["publication_date"] = "1900-01-01"+ else:+ output[0]["doi"] = "10.1234/incorrect"+ with pytest.raises(AssertionError, match="Citation output mismatch"):+ verify_batch(inputs, output, sources, expected)+++def test_repeated_normalization_observes_changed_input_without_sharing_names():+ sources, _ = corpus(1, "openalex_csl")+ source = sources[0]+ first = normalizer.normalize_citation(source)+ source["title"] = "Updated title"+ source["metadata"]["authors_csl"][0]["given"] = "Changed"+ second = normalizer.normalize_citation(source)+ assert first["csl_json"]["title"] == "Synthetic paper 0"+ assert first["authors"][0]["given"] == "Given0"+ assert second["csl_json"]["title"] == "Updated title"+ assert second["authors"][0]["given"] == "Changed"diff --git a/tests/performance/test_pytest_workload.py b/tests/performance/test_pytest_workload.pynew file mode 100644index 0000000000..207333725a--- /dev/null+++ b/tests/performance/test_pytest_workload.py@@ -0,0 +1,201 @@+"""Configuration precedence and retained diagnostics for offline workloads."""+# allow: no-sut-import — tests the performance runner, not application behavior.++import json+import sys+from pathlib import Path+from types import SimpleNamespace++import pytest++from tests.performance._shared.pytest_workload import Workload, resolve+from tests.performance._shared import pytest_workload as runner++PROFILES = {+ "ci": dict(size=2, iterations=1, warmups=0),+ "local": dict(size=10, iterations=3, warmups=1),+}+++def config(**options):+ return SimpleNamespace(getoption=lambda name: options.get(name))+++def test_cli_overrides_area_environment_and_global_profile(monkeypatch):+ monkeypatch.setenv("LDR_PERF_PROFILE", "local")+ monkeypatch.setenv("LDR_PERF_DEMO_PROFILE", "ci")+ monkeypatch.setenv("LDR_PERF_DEMO_SIZE", "7")+ settings = resolve(config(**{"--demo-size": "4"}), "demo", PROFILES)+ assert settings["profile"] == "ci"+ assert settings["size"] == 4+ assert settings["iterations"] == 1+++def test_empty_environment_preserves_profile_defaults(monkeypatch):+ monkeypatch.setenv("LDR_PERF_PROFILE", "")+ monkeypatch.setenv("LDR_PERF_DEMO_PROFILE", "")+ monkeypatch.setenv("LDR_PERF_DEMO_SIZE", "")+ settings = resolve(config(), "demo", PROFILES)+ assert settings["profile"] == "ci"+ assert settings["size"] == 2+++ "field,value",+ [("size", "0"), ("iterations", "many"), ("max-p95-ms", "nan")],+)+def test_invalid_sizes_and_budgets_fail_before_running(field, value):+ with pytest.raises(pytest.UsageError):+ resolve(config(**{f"--demo-{field}": value}), "demo", PROFILES)+++def test_failed_timing_budget_keeps_raw_samples(tmp_path, monkeypatch):+ path = tmp_path / "report.json"+ settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)+ settings.update(report=str(path), max_p95_ms=1)+ workload = Workload("demo", settings)+ clock = Clock(monkeypatch)+ workload.measure(+ "measured",+ 1,+ lambda: None,+ lambda _: clock.advance(10_000_000),+ lambda *_: {},+ )+ with pytest.raises(pytest.fail.Exception, match="p95 budget"):+ workload.finish()+ report = json.loads(path.read_text())+ assert report["budget_failures"] == ["measured[1]"]+ assert report["results"][0]["samples_ms"] == [10]+ assert report["status"] == "failed"+++def test_correctness_failure_keeps_case_diagnostics(tmp_path):+ settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)+ settings["report"] = str(tmp_path / "incorrect.json")+ workload = Workload("demo", settings)++ def verify(_state, output):+ assert output == "expected"++ with pytest.raises(AssertionError):+ workload.measure(+ "incorrect", 1, lambda: None, lambda _: "wrong", verify+ )+ workload.finish()+ report = json.loads(Path(settings["report"]).read_text())+ assert report["results"][0]["status"] == "failed"+ assert report["results"][0]["failure_phase"] == "verify"+ assert report["status"] == "failed"+ assert len(report["results"][0]["samples_ms"]) == 1+++class Clock:+ def __init__(self, monkeypatch):+ self.value = 0+ monkeypatch.setattr(runner.time, "perf_counter_ns", lambda: self.value)+ monkeypatch.setattr(+ runner.time, "perf_counter", lambda: self.value / 1_000_000_000+ )++ def advance(self, nanoseconds):+ self.value += nanoseconds+++def test_setup_checks_warmups_and_prior_result_release_are_untimed(monkeypatch):+ clock = Clock(monkeypatch)+ settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)+ settings.update(iterations=2, warmups=1)+ workload = Workload("demo", settings)+ released = []+ durations = iter((99_000_000, 5_000_000, 7_000_000))++ class Result:+ def __del__(self):+ clock.advance(50_000_000)+ released.append(True)++ def prepare():+ clock.advance(100_000_000)++ def operation(_state):+ clock.advance(next(durations))+ return Result()++ def verify(_state, _output):+ clock.advance(200_000_000)+ return {"checked": True}++ workload.measure("timed", 1, prepare, operation, verify)+ assert workload.results[0]["samples_ms"] == [5, 7]+ assert workload.results[0]["count"] == 2+ assert len(released) == 3+++def test_rates_and_scaling_use_declared_work_units(monkeypatch):+ clock = Clock(monkeypatch)+ workload = Workload(+ "demo", resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)+ )+ for size in (2, 4):+ workload.measure(+ "scaled",+ size,+ lambda: None,+ lambda _: clock.advance(size * 200),+ lambda *_: {},+ units=size,+ unit="lookup",+ )+ report = workload.finish()+ assert [point["median_ns_per_unit"] for point in report["results"]] == [+ 200,+ 200,+ ]+ assert [point["units_per_second"] for point in report["results"]] == [+ 5_000_000,+ 5_000_000,+ ]+ assert report["observed_scaling"][0]["normalized_latency_ratio"] == 1+++def test_missing_package_metadata_preserves_measurements(tmp_path, monkeypatch):+ clock = Clock(monkeypatch)+ settings = resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)+ settings["report"] = str(tmp_path / "report.json")+ workload = Workload("demo", settings)+ workload.measure(+ "measured", 1, lambda: None, lambda _: clock.advance(100), lambda *_: {}+ )++ def unavailable(_name):+ raise runner.PackageNotFoundError++ monkeypatch.setattr(runner, "version", unavailable)+ workload.finish(("optional-dependency",))+ report = json.loads(Path(settings["report"]).read_text())+ assert report["environment"]["packages"] == {"optional-dependency": None}+ assert report["results"][0]["samples_ns"] == [100]+++def test_untracked_loaded_code_is_not_stamped_with_harness_revision(+ tmp_path, monkeypatch+):+ implementation = tmp_path / "implementation.py"+ implementation.write_text("value = 42\n")+ monkeypatch.setitem(+ sys.modules,+ "workload_demo_implementation",+ SimpleNamespace(__file__=str(implementation)),+ )+ workload = Workload(+ "demo", resolve(config(**{"--demo-profile": "ci"}), "demo", PROFILES)+ )+ report = workload.finish(modules=("workload_demo_implementation",))+ loaded = report["environment"]["loaded_sources"][+ "workload_demo_implementation"+ ]+ assert loaded["path"] == str(implementation)+ assert loaded["revision"] is None+ assert len(loaded["sha256"]) == 64+ assert report["environment"]["source_revision"] is None