Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
test: fix PR formatting and cover log compressor
JerrettDavis6e9ea54a0431
diff --git a/tests/test_cli_tools.py b/tests/test_cli_tools.pyindex f74afd13b..913749bdf 100644--- a/tests/test_cli_tools.py+++ b/tests/test_cli_tools.py@@ -1,266 +1,266 @@-from __future__ import annotations--import sys-from pathlib import Path-from types import SimpleNamespace--import pytest-from click.testing import CliRunner--from headroom import binaries-from headroom.cli import tools as cli_tools-from headroom.cli.main import main---def runner() -> CliRunner:- return CliRunner()---class FakeTable:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.columns: list[str] = []- self.rows: list[tuple[object, ...]] = []-- def add_column(self, name: str) -> None:- self.columns.append(name)-- def add_row(self, *values: object) -> None:- self.rows.append(values)---class FakeConsole:- instances: list[FakeConsole] = []-- def __init__(self) -> None:- self.printed: list[object] = []- FakeConsole.instances.append(self)-- def print(self, value: object) -> None:- self.printed.append(value)---def install_fake_rich(monkeypatch: pytest.MonkeyPatch) -> None:- FakeConsole.instances.clear()- monkeypatch.setitem(sys.modules, "rich.console", SimpleNamespace(Console=FakeConsole))- monkeypatch.setitem(sys.modules, "rich.table", SimpleNamespace(Table=FakeTable))- monkeypatch.setitem(- sys.modules, "rich.markup", SimpleNamespace(escape=lambda value: f"escaped:{value}")- )---def test_exec_tool_windows_and_posix_paths(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(cli_tools.binaries, "resolve", lambda tool: Path("C:\\bin\\sg.exe"))- monkeypatch.setattr(cli_tools.os, "name", "nt", raising=False)- captured: dict[str, object] = {}-- def fake_run(cmd, check=False): # noqa: ANN001- captured["cmd"] = cmd- return SimpleNamespace(returncode=7)-- monkeypatch.setattr(cli_tools.subprocess, "run", fake_run)- with pytest.raises(SystemExit) as excinfo:- cli_tools._exec_tool("ast-grep", ["--json"])- assert excinfo.value.code == 7- assert captured["cmd"] == ["C:\\bin\\sg.exe", "--json"]-- monkeypatch.setattr(cli_tools.os, "name", "posix", raising=False)-- def fake_execv(path: str, cmd: list[str]) -> None:- raise SystemExit((path, cmd))-- monkeypatch.setattr(cli_tools.os, "execv", fake_execv)- with pytest.raises(SystemExit) as posix_exit:- cli_tools._exec_tool("ast-grep", ["--help"])- assert posix_exit.value.code == (str(Path("C:\\bin\\sg.exe")), ["C:\\bin\\sg.exe", "--help"])--- ("error", "expected"),- [- (binaries.PlatformNotSupported("unsupported"), "error: unsupported"),- (binaries.OfflineError("offline"), "Hint: run `headroom tools install`"),- (binaries.Sha256Mismatch("bad sha"), "error: bad sha"),- (binaries.BinaryFetchError("fetch failed"), "error: fetch failed"),- ],-)-def test_sg_command_reports_resolution_errors(- monkeypatch: pytest.MonkeyPatch,- runner: CliRunner,- error: Exception,- expected: str,-) -> None:- monkeypatch.setattr(cli_tools.binaries, "resolve", lambda tool: (_ for _ in ()).throw(error))- result = runner.invoke(main, ["sg", "--version"])- assert result.exit_code == 2- assert expected in result.output---def test_tools_list_renders_registry(monkeypatch: pytest.MonkeyPatch, runner: CliRunner) -> None:- install_fake_rich(monkeypatch)- monkeypatch.setattr(- cli_tools.binaries,- "detect_platform",- lambda: SimpleNamespace(key=lambda: "windows-x86_64"),- )- monkeypatch.setattr(cli_tools.binaries, "cache_dir", lambda: Path("C:\\cache"))- monkeypatch.setattr(- cli_tools.binaries,- "_registry",- lambda: {- "tools": {- "ast-grep": {- "version": "1.2.3",- "source": "github",- "assets": {"windows-x86_64": {}, "linux-x86_64-gnu": {}},- },- "python-tool": {"version": "0.1.0", "source": "pypi", "assets": {}},- }- },- )-- result = runner.invoke(main, ["tools", "list"])- assert result.exit_code == 0- console = FakeConsole.instances[-1]- assert console.printed[0] == "[dim]platform:[/dim] windows-x86_64"- assert console.printed[1] == "[dim]cache:[/dim] C:\\cache"- table = console.printed[2]- assert isinstance(table, FakeTable)- assert ("ast-grep", "1.2.3", "github", "linux-x86_64-gnu, windows-x86_64") in table.rows- assert ("python-tool", "0.1.0", "pypi", "(pypi)") in table.rows---def test_tools_doctor_json_and_table_modes(- monkeypatch: pytest.MonkeyPatch, runner: CliRunner-) -> None:- rows = [- {- "tool": "ast-grep",- "state": "cached",- "version": "1.0",- "platform": "win",- "path": "C:\\bin\\sg.exe",- },- {- "tool": "difft",- "state": "missing",- "version": "2.0",- "platform": "win",- "path": None,- "detail": "download needed <soon>",- },- ]- monkeypatch.setattr(cli_tools.binaries, "status", lambda: rows)-- json_result = runner.invoke(main, ["tools", "doctor", "--json"])- assert json_result.exit_code == 1- assert '"tool": "ast-grep"' in json_result.output- assert '"state": "missing"' in json_result.output-- install_fake_rich(monkeypatch)- table_result = runner.invoke(main, ["tools", "doctor"])- assert table_result.exit_code == 1- console = FakeConsole.instances[-1]- table = console.printed[0]- assert isinstance(table, FakeTable)- assert ("ast-grep", "[green]cached[/green]", "1.0", "win", "C:\\bin\\sg.exe") in table.rows- assert ("difft", "[yellow]missing[/yellow]", "2.0", "win", "-") in table.rows- assert console.printed[1] == "[dim]difft:[/dim] escaped:download needed <soon>"---def test_tools_install_covers_unknown_pypi_force_and_failures(- monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path-) -> None:- cached_path = tmp_path / "cached-tool.exe"- cached_path.write_text("x", encoding="utf-8")- monkeypatch.setattr(- cli_tools.binaries,- "_registry",- lambda: {- "tools": {- "known": {"version": "1.0"},- "pypi_tool": {"version": "2.0"},- "broken": {"version": "3.0"},- }- },- )- monkeypatch.setattr(cli_tools.binaries, "_is_pypi_tool", lambda name: name == "pypi_tool")- monkeypatch.setattr(- cli_tools.binaries,- "_path_lookup",- lambda name: Path("C:\\Python\\Scripts\\pypi_tool.exe") if name == "pypi_tool" else None,- )- monkeypatch.setattr(- cli_tools.binaries,- "detect_platform",- lambda: SimpleNamespace(key=lambda: "windows-x86_64"),- )- monkeypatch.setattr(cli_tools.binaries, "_cached_path", lambda name, version, plat: cached_path)- monkeypatch.setattr(- cli_tools.binaries,- "resolve",- lambda name: (_ for _ in ()).throw(binaries.OfflineError("offline"))- if name == "broken"- else Path(f"C:\\cache\\{name}.exe"),- )-- result = runner.invoke(- main,- [- "tools",- "install",- "--tool",- "missing",- "--tool",- "pypi_tool",- "--tool",- "known",- "--tool",- "broken",- "--force",- ],- )-- assert result.exit_code == 1- assert "unknown tool 'missing'; skipping" in result.output- assert "pypi_tool: on PATH at C:\\Python\\Scripts\\pypi_tool.exe (pypi wheel)" in result.output- assert "known: installed" in result.output- assert "broken: offline" in result.output- assert not cached_path.exists()---def test_tools_install_reports_missing_pypi_and_cached_unlink_failure(- monkeypatch: pytest.MonkeyPatch, runner: CliRunner-) -> None:- problem_path = Path("C:\\cache\\locked.exe")- monkeypatch.setattr(- cli_tools.binaries,- "_registry",- lambda: {"tools": {"pypi_tool": {"version": "1.0"}, "known": {"version": "2.0"}}},- )- monkeypatch.setattr(cli_tools.binaries, "_is_pypi_tool", lambda name: name == "pypi_tool")- monkeypatch.setattr(cli_tools.binaries, "_path_lookup", lambda name: None)- monkeypatch.setattr(- cli_tools.binaries,- "detect_platform",- lambda: SimpleNamespace(key=lambda: "windows-x86_64"),- )- monkeypatch.setattr(- cli_tools.binaries, "_cached_path", lambda name, version, plat: problem_path- )- monkeypatch.setattr(Path, "exists", lambda self: self == problem_path)-- def fake_unlink(self) -> None:- raise OSError("locked")-- monkeypatch.setattr(Path, "unlink", fake_unlink)- monkeypatch.setattr(cli_tools.binaries, "resolve", lambda name: Path(f"C:\\cache\\{name}.exe"))-- result = runner.invoke(- main,- ["tools", "install", "--tool", "pypi_tool", "--tool", "known", "--force"],- )-- assert result.exit_code == 1- assert "pypi_tool: not on PATH" in result.output- assert "known: failed to remove cached binary: locked" in result.output+from __future__ import annotations++import sys+from pathlib import Path+from types import SimpleNamespace++import pytest+from click.testing import CliRunner++from headroom import binaries+from headroom.cli import tools as cli_tools+from headroom.cli.main import main+++def runner() -> CliRunner:+ return CliRunner()+++class FakeTable:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ self.columns: list[str] = []+ self.rows: list[tuple[object, ...]] = []++ def add_column(self, name: str) -> None:+ self.columns.append(name)++ def add_row(self, *values: object) -> None:+ self.rows.append(values)+++class FakeConsole:+ instances: list[FakeConsole] = []++ def __init__(self) -> None:+ self.printed: list[object] = []+ FakeConsole.instances.append(self)++ def print(self, value: object) -> None:+ self.printed.append(value)+++def install_fake_rich(monkeypatch: pytest.MonkeyPatch) -> None:+ FakeConsole.instances.clear()+ monkeypatch.setitem(sys.modules, "rich.console", SimpleNamespace(Console=FakeConsole))+ monkeypatch.setitem(sys.modules, "rich.table", SimpleNamespace(Table=FakeTable))+ monkeypatch.setitem(+ sys.modules, "rich.markup", SimpleNamespace(escape=lambda value: f"escaped:{value}")+ )+++def test_exec_tool_windows_and_posix_paths(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(cli_tools.binaries, "resolve", lambda tool: Path("C:\\bin\\sg.exe"))+ monkeypatch.setattr(cli_tools.os, "name", "nt", raising=False)+ captured: dict[str, object] = {}++ def fake_run(cmd, check=False): # noqa: ANN001+ captured["cmd"] = cmd+ return SimpleNamespace(returncode=7)++ monkeypatch.setattr(cli_tools.subprocess, "run", fake_run)+ with pytest.raises(SystemExit) as excinfo:+ cli_tools._exec_tool("ast-grep", ["--json"])+ assert excinfo.value.code == 7+ assert captured["cmd"] == ["C:\\bin\\sg.exe", "--json"]++ monkeypatch.setattr(cli_tools.os, "name", "posix", raising=False)++ def fake_execv(path: str, cmd: list[str]) -> None:+ raise SystemExit((path, cmd))++ monkeypatch.setattr(cli_tools.os, "execv", fake_execv)+ with pytest.raises(SystemExit) as posix_exit:+ cli_tools._exec_tool("ast-grep", ["--help"])+ assert posix_exit.value.code == (str(Path("C:\\bin\\sg.exe")), ["C:\\bin\\sg.exe", "--help"])+++ ("error", "expected"),+ [+ (binaries.PlatformNotSupported("unsupported"), "error: unsupported"),+ (binaries.OfflineError("offline"), "Hint: run `headroom tools install`"),+ (binaries.Sha256Mismatch("bad sha"), "error: bad sha"),+ (binaries.BinaryFetchError("fetch failed"), "error: fetch failed"),+ ],+)+def test_sg_command_reports_resolution_errors(+ monkeypatch: pytest.MonkeyPatch,+ runner: CliRunner,+ error: Exception,+ expected: str,+) -> None:+ monkeypatch.setattr(cli_tools.binaries, "resolve", lambda tool: (_ for _ in ()).throw(error))+ result = runner.invoke(main, ["sg", "--version"])+ assert result.exit_code == 2+ assert expected in result.output+++def test_tools_list_renders_registry(monkeypatch: pytest.MonkeyPatch, runner: CliRunner) -> None:+ install_fake_rich(monkeypatch)+ monkeypatch.setattr(+ cli_tools.binaries,+ "detect_platform",+ lambda: SimpleNamespace(key=lambda: "windows-x86_64"),+ )+ monkeypatch.setattr(cli_tools.binaries, "cache_dir", lambda: Path("C:\\cache"))+ monkeypatch.setattr(+ cli_tools.binaries,+ "_registry",+ lambda: {+ "tools": {+ "ast-grep": {+ "version": "1.2.3",+ "source": "github",+ "assets": {"windows-x86_64": {}, "linux-x86_64-gnu": {}},+ },+ "python-tool": {"version": "0.1.0", "source": "pypi", "assets": {}},+ }+ },+ )++ result = runner.invoke(main, ["tools", "list"])+ assert result.exit_code == 0+ console = FakeConsole.instances[-1]+ assert console.printed[0] == "[dim]platform:[/dim] windows-x86_64"+ assert console.printed[1] == "[dim]cache:[/dim] C:\\cache"+ table = console.printed[2]+ assert isinstance(table, FakeTable)+ assert ("ast-grep", "1.2.3", "github", "linux-x86_64-gnu, windows-x86_64") in table.rows+ assert ("python-tool", "0.1.0", "pypi", "(pypi)") in table.rows+++def test_tools_doctor_json_and_table_modes(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner+) -> None:+ rows = [+ {+ "tool": "ast-grep",+ "state": "cached",+ "version": "1.0",+ "platform": "win",+ "path": "C:\\bin\\sg.exe",+ },+ {+ "tool": "difft",+ "state": "missing",+ "version": "2.0",+ "platform": "win",+ "path": None,+ "detail": "download needed <soon>",+ },+ ]+ monkeypatch.setattr(cli_tools.binaries, "status", lambda: rows)++ json_result = runner.invoke(main, ["tools", "doctor", "--json"])+ assert json_result.exit_code == 1+ assert '"tool": "ast-grep"' in json_result.output+ assert '"state": "missing"' in json_result.output++ install_fake_rich(monkeypatch)+ table_result = runner.invoke(main, ["tools", "doctor"])+ assert table_result.exit_code == 1+ console = FakeConsole.instances[-1]+ table = console.printed[0]+ assert isinstance(table, FakeTable)+ assert ("ast-grep", "[green]cached[/green]", "1.0", "win", "C:\\bin\\sg.exe") in table.rows+ assert ("difft", "[yellow]missing[/yellow]", "2.0", "win", "-") in table.rows+ assert console.printed[1] == "[dim]difft:[/dim] escaped:download needed <soon>"+++def test_tools_install_covers_unknown_pypi_force_and_failures(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path+) -> None:+ cached_path = tmp_path / "cached-tool.exe"+ cached_path.write_text("x", encoding="utf-8")+ monkeypatch.setattr(+ cli_tools.binaries,+ "_registry",+ lambda: {+ "tools": {+ "known": {"version": "1.0"},+ "pypi_tool": {"version": "2.0"},+ "broken": {"version": "3.0"},+ }+ },+ )+ monkeypatch.setattr(cli_tools.binaries, "_is_pypi_tool", lambda name: name == "pypi_tool")+ monkeypatch.setattr(+ cli_tools.binaries,+ "_path_lookup",+ lambda name: Path("C:\\Python\\Scripts\\pypi_tool.exe") if name == "pypi_tool" else None,+ )+ monkeypatch.setattr(+ cli_tools.binaries,+ "detect_platform",+ lambda: SimpleNamespace(key=lambda: "windows-x86_64"),+ )+ monkeypatch.setattr(cli_tools.binaries, "_cached_path", lambda name, version, plat: cached_path)+ monkeypatch.setattr(+ cli_tools.binaries,+ "resolve",+ lambda name: (_ for _ in ()).throw(binaries.OfflineError("offline"))+ if name == "broken"+ else Path(f"C:\\cache\\{name}.exe"),+ )++ result = runner.invoke(+ main,+ [+ "tools",+ "install",+ "--tool",+ "missing",+ "--tool",+ "pypi_tool",+ "--tool",+ "known",+ "--tool",+ "broken",+ "--force",+ ],+ )++ assert result.exit_code == 1+ assert "unknown tool 'missing'; skipping" in result.output+ assert "pypi_tool: on PATH at C:\\Python\\Scripts\\pypi_tool.exe (pypi wheel)" in result.output+ assert "known: installed" in result.output+ assert "broken: offline" in result.output+ assert not cached_path.exists()+++def test_tools_install_reports_missing_pypi_and_cached_unlink_failure(+ monkeypatch: pytest.MonkeyPatch, runner: CliRunner+) -> None:+ problem_path = Path("C:\\cache\\locked.exe")+ monkeypatch.setattr(+ cli_tools.binaries,+ "_registry",+ lambda: {"tools": {"pypi_tool": {"version": "1.0"}, "known": {"version": "2.0"}}},+ )+ monkeypatch.setattr(cli_tools.binaries, "_is_pypi_tool", lambda name: name == "pypi_tool")+ monkeypatch.setattr(cli_tools.binaries, "_path_lookup", lambda name: None)+ monkeypatch.setattr(+ cli_tools.binaries,+ "detect_platform",+ lambda: SimpleNamespace(key=lambda: "windows-x86_64"),+ )+ monkeypatch.setattr(+ cli_tools.binaries, "_cached_path", lambda name, version, plat: problem_path+ )+ monkeypatch.setattr(Path, "exists", lambda self: self == problem_path)++ def fake_unlink(self) -> None:+ raise OSError("locked")++ monkeypatch.setattr(Path, "unlink", fake_unlink)+ monkeypatch.setattr(cli_tools.binaries, "resolve", lambda name: Path(f"C:\\cache\\{name}.exe"))++ result = runner.invoke(+ main,+ ["tools", "install", "--tool", "pypi_tool", "--tool", "known", "--force"],+ )++ assert result.exit_code == 1+ assert "pypi_tool: not on PATH" in result.output+ assert "known: failed to remove cached binary: locked" in result.outputdiff --git a/tests/test_memory_handler_native_ops.py b/tests/test_memory_handler_native_ops.pyindex e75ce3c04..ef80a5a0d 100644--- a/tests/test_memory_handler_native_ops.py+++ b/tests/test_memory_handler_native_ops.py@@ -1,1395 +1,1395 @@-from __future__ import annotations--import asyncio-import json-import sys-from pathlib import Path-from types import SimpleNamespace--import pytest--from headroom.proxy import memory_handler as memory_handler_module-from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler---def handler(tmp_path: Path) -> MemoryHandler:- return MemoryHandler(- MemoryConfig(- enabled=False,- use_native_tool=True,- native_memory_dir=str(tmp_path / "native"),- ),- agent_type="codex",- )---class FakeBackend:- def __init__(self) -> None:- self.search_results: list[object] = []- self.saved: list[dict[str, object]] = []- self.updated: list[dict[str, object]] = []- self.deleted: list[str] = []- self.raise_on: str | None = None-- async def search_memories(self, **kwargs): # noqa: ANN003- if self.raise_on == "search":- raise RuntimeError("search failed")- return self.search_results-- async def save_memory(self, **kwargs): # noqa: ANN003- if self.raise_on == "save":- raise RuntimeError("save failed")- self.saved.append(kwargs)- return SimpleNamespace(id=f"mem-{len(self.saved)}", content=kwargs["content"])-- async def update_memory(self, **kwargs): # noqa: ANN003- if self.raise_on == "update":- raise RuntimeError("update failed")- self.updated.append(kwargs)- return SimpleNamespace(id=kwargs["memory_id"])-- async def delete_memory(self, memory_id: str): # noqa: ANN201- if self.raise_on == "delete":- raise RuntimeError("delete failed")- self.deleted.append(memory_id)- return True---def make_result(- memory_id: str,- content: str,- *,- score: float = 0.9,- metadata: dict[str, object] | None = None,- related_entities: list[str] | None = None,- created_at: str | None = None,- importance: float = 0.5,-) -> object:- return SimpleNamespace(- memory=SimpleNamespace(- id=memory_id,- content=content,- metadata=metadata or {},- created_at=created_at,- importance=importance,- ),- score=score,- related_entities=related_entities or [],- )---def test_resolve_native_path_blocks_traversal(handler: MemoryHandler) -> None:- resolved = handler._resolve_native_path("/memories/topic.txt", "u1")- assert resolved.name == "topic.txt"- assert "u1" in str(resolved)-- with pytest.raises(ValueError, match="Path traversal detected"):- handler._resolve_native_path("/memories/../escape.txt", "u1")---def test_native_view_lists_directory_and_reads_files(handler: MemoryHandler) -> None:- root = handler._resolve_native_path("/memories", "u1")- (root / "alpha.txt").write_text("line1\nline2\nline3", encoding="utf-8")- (root / "nested").mkdir()- (root / "nested" / "beta.txt").write_text("nested", encoding="utf-8")- (root / ".hidden").write_text("skip", encoding="utf-8")- (root / "node_modules").mkdir()-- listing = handler._native_view({"path": "/memories"}, "u1")- assert "/memories/alpha.txt" in listing- assert "/memories/nested/beta.txt" in listing- assert ".hidden" not in listing- assert "/memories/node_modules" not in listing-- file_view = handler._native_view({"path": "/memories/alpha.txt", "view_range": [2, 3]}, "u1")- assert "2\tline2" in file_view- assert "3\tline3" in file_view---def test_native_view_handles_missing_paths_and_latin1(handler: MemoryHandler) -> None:- missing = handler._native_view({"path": "/memories/missing.txt"}, "u1")- assert "does not exist" in missing-- latin_path = handler._resolve_native_path("/memories/latin.txt", "u1")- latin_path.write_bytes("caf\xe9".encode("latin-1"))- viewed = handler._native_view({"path": "/memories/latin.txt"}, "u1")- assert "cafe" not in viewed- assert "café" in viewed---def test_native_create_insert_delete_and_rename(handler: MemoryHandler) -> None:- assert handler._native_create(- {"path": "/memories/note.txt", "file_text": "a\nb"}, "u1"- ).startswith("File created successfully")- assert handler._native_create(- {"path": "/memories/note.txt", "file_text": "dup"}, "u1"- ).startswith("Error: File /memories/note.txt already exists")-- inserted = handler._native_insert(- {"path": "/memories/note.txt", "insert_line": 1, "insert_text": "middle"},- "u1",- )- assert inserted == "The file /memories/note.txt has been edited."- assert "middle" in handler._resolve_native_path("/memories/note.txt", "u1").read_text(- encoding="utf-8"- )-- renamed = handler._native_rename(- {"old_path": "/memories/note.txt", "new_path": "/memories/archive/renamed.txt"},- "u1",- )- assert renamed == "Successfully renamed /memories/note.txt to /memories/archive/renamed.txt"-- deleted = handler._native_delete_file({"path": "/memories/archive"}, "u1")- assert deleted == "Successfully deleted /memories/archive"---def test_native_insert_validates_range_and_path(handler: MemoryHandler) -> None:- assert (- handler._native_insert({"insert_line": 0, "insert_text": "x"}, "u1")- == "Error: path is required"- )- assert "does not exist" in handler._native_insert(- {"path": "/memories/missing.txt", "insert_line": 0, "insert_text": "x"},- "u1",- )-- note = handler._resolve_native_path("/memories/note.txt", "u1")- note.write_text("a\nb", encoding="utf-8")- invalid = handler._native_insert(- {"path": "/memories/note.txt", "insert_line": 4, "insert_text": "x"},- "u1",- )- assert "Invalid `insert_line` parameter: 4" in invalid---def test_native_str_replace_covers_missing_multiple_and_success(handler: MemoryHandler) -> None:- note = handler._resolve_native_path("/memories/note.txt", "u1")- note.write_text("hello\nhello\nworld", encoding="utf-8")-- assert (- handler._native_str_replace({"old_str": "hello", "new_str": "bye"}, "u1")- == "Error: path is required"- )- assert (- handler._native_str_replace({"path": "/memories/note.txt", "new_str": "bye"}, "u1")- == "Error: old_str is required"- )-- multiple = handler._native_str_replace(- {"path": "/memories/note.txt", "old_str": "hello", "new_str": "bye"},- "u1",- )- assert "Multiple occurrences of old_str `hello` in lines: 1, 2" in multiple-- note.write_text("hello\nworld", encoding="utf-8")- missing = handler._native_str_replace(- {"path": "/memories/note.txt", "old_str": "nope", "new_str": "bye"},- "u1",- )- assert "did not appear verbatim" in missing-- success = handler._native_str_replace(- {"path": "/memories/note.txt", "old_str": "hello", "new_str": "bye"},- "u1",- )- assert "The memory file has been edited." in success- assert "bye" in note.read_text(encoding="utf-8")---def test_native_delete_and_rename_validate_inputs(handler: MemoryHandler) -> None:- assert handler._native_delete_file({}, "u1") == "Error: path is required"- assert "does not exist" in handler._native_delete_file({"path": "/memories/missing.txt"}, "u1")-- assert (- handler._native_rename({"new_path": "/memories/new.txt"}, "u1")- == "Error: old_path is required"- )- assert (- handler._native_rename({"old_path": "/memories/old.txt"}, "u1")- == "Error: new_path is required"- )- assert "does not exist" in handler._native_rename(- {"old_path": "/memories/old.txt", "new_path": "/memories/new.txt"},- "u1",- )-- old = handler._resolve_native_path("/memories/old.txt", "u1")- new = handler._resolve_native_path("/memories/new.txt", "u1")- old.write_text("x", encoding="utf-8")- new.write_text("y", encoding="utf-8")- assert (- handler._native_rename(- {"old_path": "/memories/old.txt", "new_path": "/memories/new.txt"},- "u1",- )- == "Error: The destination /memories/new.txt already exists"- )---async def test_execute_native_memory_tool_dispatches_and_wraps_errors(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- handler._backend = object()- called: list[tuple[str, dict[str, object], str]] = []-- async def fake_ensure_initialized() -> None:- return None-- async def fake_view(input_data, user_id): # noqa: ANN001- called.append(("view", input_data, user_id))- return "viewed"-- async def fake_create(input_data, user_id): # noqa: ANN001- called.append(("create", input_data, user_id))- return "created"-- monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)- monkeypatch.setattr(handler, "_native_view_semantic", fake_view)- monkeypatch.setattr(handler, "_native_create_semantic", fake_create)-- assert await handler._execute_native_memory_tool({"command": "view"}, "u1") == "viewed"- assert await handler._execute_native_memory_tool({"command": "create"}, "u1") == "created"- assert (- await handler._execute_native_memory_tool({"command": "bad"}, "u1")- == "Error: Unknown command 'bad'"- )-- async def boom(input_data, user_id): # noqa: ANN001- raise RuntimeError("oops")-- monkeypatch.setattr(handler, "_native_view_semantic", boom)- assert await handler._execute_native_memory_tool({"command": "view"}, "u1") == "Error: oops"- assert [entry[0] for entry in called] == ["view", "create"]---async def test_semantic_search_recent_all_and_overview(handler: MemoryHandler) -> None:- backend = FakeBackend()- handler._backend = backend- backend.search_results = [- make_result(- "m1",- "Alice likes pizza and pasta",- score=0.91,- related_entities=["Alice", "pizza"],- created_at="2026-04-22",- ),- make_result("m2", "Bob prefers ramen", score=0.83),- ]-- search_text = await handler._semantic_search("pizza", "u1")- assert "Found 2 memories matching 'pizza'" in search_text- assert "[91% match] Alice likes pizza and pasta" in search_text- assert "Related: Alice, pizza" in search_text-- recent_text = await handler._get_recent_memories("u1", limit=2)- assert "Recent memories:" in recent_text- assert "(2026-04-22)" in recent_text-- all_text = await handler._list_all_memories("u1", limit=2)- assert "Showing up to 2 memories:" in all_text- assert "Showing first 2" in all_text-- overview = await handler._get_memory_overview("u1")- assert "Memory System (2 memories stored)" in overview- assert "view /memories/search/<your query>" in overview---async def test_semantic_helpers_handle_empty_backend_and_errors(handler: MemoryHandler) -> None:- assert await handler._semantic_search("x", "u1") == "Error: Memory backend not initialized"- assert await handler._get_recent_memories("u1") == "Error: Memory backend not initialized"- assert await handler._list_all_memories("u1") == "Error: Memory backend not initialized"- assert await handler._get_memory_overview("u1") == "Error: Memory backend not initialized"-- backend = FakeBackend()- handler._backend = backend- assert "No memories found matching 'x'" in await handler._semantic_search("x", "u1")- assert "No memories stored yet." in await handler._list_all_memories("u1")- assert "No memories stored yet." in await handler._get_recent_memories("u1")-- backend.raise_on = "search"- assert "Error searching memories: search failed" == await handler._semantic_search("x", "u1")- assert "Error getting recent memories: search failed" == await handler._get_recent_memories(- "u1"- )- assert "Error listing memories: search failed" == await handler._list_all_memories("u1")- overview = await handler._get_memory_overview("u1")- assert "📁 Memory System" in overview- assert "To SEARCH memories" in overview---async def test_native_view_semantic_routes_paths(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- seen: list[tuple[str, object]] = []-- async def fake_search(query, user_id, top_k=5): # noqa: ANN001- seen.append(("search", query))- return "search-result"-- async def fake_recent(user_id, limit=10): # noqa: ANN001- seen.append(("recent", limit))- return "recent-result"-- async def fake_all(user_id, limit=20): # noqa: ANN001- seen.append(("all", limit))- return "all-result"-- async def fake_overview(user_id): # noqa: ANN001- seen.append(("overview", user_id))- return "overview-result"-- monkeypatch.setattr(handler, "_semantic_search", fake_search)- monkeypatch.setattr(handler, "_get_recent_memories", fake_recent)- monkeypatch.setattr(handler, "_list_all_memories", fake_all)- monkeypatch.setattr(handler, "_get_memory_overview", fake_overview)-- assert (- await handler._native_view_semantic({"path": "/memories/search/pizza"}, "u1")- == "search-result"- )- assert (- await handler._native_view_semantic({"path": "/memories/recent"}, "u1") == "recent-result"- )- assert await handler._native_view_semantic({"path": "/memories/all"}, "u1") == "all-result"- assert await handler._native_view_semantic({"path": "/memories"}, "u1") == "overview-result"- assert (- await handler._native_view_semantic({"path": "/memories/work/projects"}, "u1")- == "search-result"- )- assert (await handler._native_view_semantic({"path": "/memories/search/"}, "u1")).startswith(- "Error: Please provide a search query"- )- assert seen == [- ("search", "pizza"),- ("recent", 10),- ("all", 20),- ("overview", "u1"),- ("search", "work projects"),- ]---async def test_native_semantic_create_append_delete_and_rename(handler: MemoryHandler) -> None:- backend = FakeBackend()- handler._backend = backend-- assert await handler._native_create_semantic({}, "u1") == "Error: path is required"- assert (- await handler._native_create_semantic({"path": "/memories/topic.txt"}, "u1")- == "Error: file_text is required (the memory content)"- )-- created = await handler._native_create_semantic(- {"path": "/memories/topic.txt", "file_text": "prefers pizza"},- "u1",- )- assert created == "File created successfully at: /memories/topic.txt"- assert backend.saved[-1]["metadata"] == {- "virtual_path": "/memories/topic.txt",- "topic": "topic",- }-- assert await handler._native_append_semantic({}, "u1") == "Error: path is required"- assert (- await handler._native_append_semantic({"path": "/memories/topic.txt"}, "u1")- == "Error: insert_text is required"- )- appended = await handler._native_append_semantic(- {"path": "/memories/topic.txt", "insert_text": "and pasta"},- "u1",- )- assert appended == "The file /memories/topic.txt has been edited."- assert backend.saved[-1]["metadata"]["appended"] is True-- backend.search_results = [- make_result(- "m1", "prefers pizza", metadata={"virtual_path": "/memories/topic.txt"}, score=0.6- ),- make_result("m2", "prefers pasta", metadata={}, score=0.91),- ]- deleted = await handler._native_delete_semantic({"path": "/memories/topic.txt"}, "u1")- assert deleted == "Successfully deleted /memories/topic.txt"- assert backend.deleted == ["m1", "m2"]-- backend.search_results = [- make_result(- "m3", "old content", metadata={"virtual_path": "/memories/old.txt"}, importance=0.7- )- ]- renamed = await handler._native_rename_semantic(- {"old_path": "/memories/old.txt", "new_path": "/memories/new/topic.txt"},- "u1",- )- assert renamed == "Successfully renamed /memories/old.txt to /memories/new/topic.txt"- assert backend.deleted[-1] == "m3"- assert backend.saved[-1]["metadata"] == {- "virtual_path": "/memories/new/topic.txt",- "topic": "new_topic",- }---async def test_native_semantic_update_delete_rename_and_backend_errors(- handler: MemoryHandler,-) -> None:- backend = FakeBackend()- handler._backend = backend-- assert await handler._native_update_semantic({}, "u1") == "Error: path is required"- assert (- await handler._native_update_semantic({"path": "/memories/t.txt"}, "u1")- == "Error: old_str is required"- )-- backend.search_results = [- make_result("m1", "hello hello world", metadata={"virtual_path": "/memories/t.txt"})- ]- multi = await handler._native_update_semantic(- {"path": "/memories/t.txt", "old_str": "hello", "new_str": "bye"},- "u1",- )- assert "Multiple occurrences of old_str `hello`" in multi-- backend.search_results = [- make_result("m1", "hello world", metadata={"virtual_path": "/memories/t.txt"})- ]- edited = await handler._native_update_semantic(- {"path": "/memories/t.txt", "old_str": "hello", "new_str": "bye"},- "u1",- )- assert "The memory file has been edited." in edited- assert backend.updated[-1]["new_content"] == "bye world"-- class NoUpdateBackend:- def __init__(self) -> None:- self.search_results: list[object] = []- self.saved: list[dict[str, object]] = []- self.deleted: list[str] = []-- async def search_memories(self, **kwargs): # noqa: ANN003- return self.search_results-- async def delete_memory(self, memory_id: str): # noqa: ANN201- self.deleted.append(memory_id)- return True-- async def save_memory(self, **kwargs): # noqa: ANN003- self.saved.append(kwargs)- return SimpleNamespace(id=f"mem-{len(self.saved)}")-- no_update_backend = NoUpdateBackend()- no_update_backend.search_results = [- make_result("m2", "alpha beta", metadata={"virtual_path": "/memories/t.txt"})- ]- handler._backend = no_update_backend- fallback = await handler._native_update_semantic(- {"path": "/memories/t.txt", "old_str": "alpha", "new_str": "omega"},- "u1",- )- assert "The memory file has been edited." in fallback- assert no_update_backend.deleted[-1] == "m2"- assert no_update_backend.saved[-1]["content"] == "omega beta"-- backend = FakeBackend()- handler._backend = backend- assert await handler._native_delete_semantic({}, "u1") == "Error: path is required"- assert await handler._native_rename_semantic({}, "u1") == "Error: old_path is required"- assert (- await handler._native_rename_semantic({"old_path": "/memories/a.txt"}, "u1")- == "Error: new_path is required"- )- assert (- await handler._native_delete_semantic({"path": "/memories/x.txt"}, "u1")- == "Error: The path /memories/x.txt does not exist"- )- assert (- await handler._native_rename_semantic(- {"old_path": "/memories/x.txt", "new_path": "/memories/y.txt"},- "u1",- )- == "Error: The path /memories/x.txt does not exist"- )-- backend.search_results = [- make_result("m9", "content", metadata={"virtual_path": "/memories/other.txt"}, score=0.1)- ]- assert (- await handler._native_delete_semantic({"path": "/memories/x.txt"}, "u1")- == "Error: The path /memories/x.txt does not exist"- )- assert (- await handler._native_rename_semantic(- {"old_path": "/memories/x.txt", "new_path": "/memories/y.txt"},- "u1",- )- == "Error: The path /memories/x.txt does not exist"- )-- backend.raise_on = "search"- assert (- await handler._native_create_semantic(- {"path": "/memories/topic.txt", "file_text": "content"},- "u1",- )- == "File created successfully at: /memories/topic.txt"- )- backend.raise_on = "save"- assert (- await handler._native_create_semantic(- {"path": "/memories/topic.txt", "file_text": "content"}, "u1"- )- ).startswith("Error: ")- assert (- await handler._native_append_semantic(- {"path": "/memories/topic.txt", "insert_text": "content"}, "u1"- )- ).startswith("Error: ")- backend.raise_on = "search"- assert (- await handler._native_update_semantic(- {"path": "/memories/t.txt", "old_str": "a", "new_str": "b"}, "u1"- )- ).startswith("Error: ")- assert (await handler._native_delete_semantic({"path": "/memories/x.txt"}, "u1")).startswith(- "Error: "- )- assert (- await handler._native_rename_semantic(- {"old_path": "/memories/x.txt", "new_path": "/memories/y.txt"},- "u1",- )- ).startswith("Error: ")---async def test_execute_search_update_delete_and_handler_status(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- backend = FakeBackend()- handler._backend = backend-- assert json.loads(await handler._execute_search({}, "u1")) == {- "status": "error",- "error": "query is required",- }-- backend.search_results = [- make_result("m1", "pizza", score=0.9123, related_entities=["food", "italy"])- ]- search_payload = json.loads(- await handler._execute_search(- {"query": "pizza", "top_k": 3, "include_related": False, "entities": ["food"]},- "u1",- )- )- assert search_payload["status"] == "found"- assert search_payload["count"] == 1- assert search_payload["memories"][0] == {- "id": "m1",- "content": "pizza",- "score": 0.912,- "entities": ["food", "italy"],- }-- assert json.loads(await handler._execute_update({}, "u1")) == {- "status": "error",- "error": "memory_id is required",- }- assert json.loads(await handler._execute_update({"memory_id": "m1"}, "u1")) == {- "status": "error",- "error": "new_content is required",- }-- backend.search_results = [make_result("m1", "old content")]- update_payload = json.loads(- await handler._execute_update(- {"memory_id": "m1", "new_content": "new content", "reason": "cleanup"},- "u1",- provider="openai",- )- )- assert update_payload == {"status": "updated", "memory_id": "m1"}- assert backend.updated[-1]["new_content"] == "new content"-- class NoUpdateBackend:- def __init__(self) -> None:- self.deleted: list[str] = []- self.saved: list[dict[str, object]] = []-- async def delete_memory(self, memory_id: str): # noqa: ANN201- self.deleted.append(memory_id)- return True-- async def save_memory(self, **kwargs): # noqa: ANN003- self.saved.append(kwargs)- return SimpleNamespace(id="m2")-- no_update_backend = NoUpdateBackend()- handler._backend = no_update_backend- update_fallback = json.loads(- await handler._execute_update({"memory_id": "m1", "new_content": "replacement"}, "u1")- )- assert update_fallback == {- "status": "updated",- "memory_id": "m2",- "note": "Replaced via delete+save",- }- assert no_update_backend.deleted == ["m1"]-- handler._backend = backend- assert json.loads(await handler._execute_delete({}, "u1")) == {- "status": "error",- "error": "memory_id is required",- }- delete_payload = json.loads(await handler._execute_delete({"memory_id": "m1"}, "u1"))- assert delete_payload == {"status": "deleted", "memory_id": "m1"}-- assert handler.health_status() == {- "enabled": False,- "backend": "local",- "initialized": False,- "native_tool": True,- "bridge_enabled": False,- }-- seen = {"count": 0}-- async def fake_ensure_initialized() -> None:- seen["count"] += 1-- monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)- await handler.ensure_initialized()- assert seen["count"] == 1---async def test_warmup_embedder_and_close(handler: MemoryHandler) -> None:- assert await handler.warmup_embedder() is False-- class FakeEmbedder:- def __init__(self) -> None:- self.calls: list[str] = []-- async def embed(self, value: str) -> None:- self.calls.append(value)-- embedder = FakeEmbedder()- handler._initialized = True- handler._backend = SimpleNamespace(_hierarchical_memory=SimpleNamespace(_embedder=embedder))- assert await handler.warmup_embedder() is True- assert embedder.calls == ["warmup"]-- handler._backend = SimpleNamespace(- _hierarchical_memory=SimpleNamespace(_embedder=SimpleNamespace())- )- assert await handler.warmup_embedder() is False-- handler._backend = SimpleNamespace(close=lambda: None)- await handler.close()- assert handler.backend is None- assert handler.initialized is False---async def test_execute_memory_tool_save_and_background_dedup(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- backend = FakeBackend()- handler._backend = backend-- assert json.loads(await handler._execute_memory_tool("unknown", {}, "u1")) == {- "error": "Unknown tool: unknown"- }- assert json.loads(await handler._execute_memory_tool("memory_save", {}, "u1")) == {- "status": "error",- "error": "content is required",- }-- created_tasks: list[object] = []-- def fake_create_task(coro): # noqa: ANN001- created_tasks.append(coro)- coro.close()- return SimpleNamespace()-- monkeypatch.setattr("headroom.proxy.memory_handler.asyncio.create_task", fake_create_task)- backend.search_results = [- make_result(- "other",- "Very similar memory content " * 5,- score=0.8,- metadata={"source_agent": "claude"},- ),- make_result("mem-1", "self result", score=0.99),- ]-- saved = json.loads(- await handler._execute_memory_tool(- "memory_save",- {- "content": "Useful fact",- "importance": 0.7,- "facts": ["fact"],- "entities": ["entity"],- "extracted_entities": ["entity"],- "relationships": ["rel"],- "extracted_relationships": ["rel"],- },- "u1",- provider="openai",- )- )- assert saved["status"] == "saved"- assert saved["memory_id"] == "mem-1"- assert "Similar memory exists" in saved["note"]- assert "saved by claude" in saved["note"]- assert backend.saved[-1]["metadata"]["source_provider"] == "openai"- assert len(created_tasks) == 1-- backend.raise_on = "save"- errored = json.loads(await handler._execute_memory_tool("memory_save", {"content": "x"}, "u1"))- assert errored == {"status": "error", "error": "save failed"}---async def test_execute_save_handles_search_failure_and_background_dedup_filters(- handler: MemoryHandler,-) -> None:- backend = FakeBackend()- handler._backend = backend-- backend.raise_on = "search"- saved = json.loads(await handler._execute_save({"content": "Useful fact"}, "u1"))- assert saved == {"status": "saved", "memory_id": "mem-1", "content": "Useful fact"}-- backend.raise_on = None- similar = [- make_result("mem-1", "same", score=0.99),- make_result("old-1", "duplicate", score=0.95, metadata={}),- make_result("old-2", "already handled", score=0.99, metadata={"superseded_by": "new"}),- make_result("old-3", "too low", score=0.5, metadata={}),- ]- await handler._background_dedup("mem-1", similar, "u1")- assert backend.deleted == ["old-1"]-- backend.raise_on = "delete"- await handler._background_dedup("mem-1", [make_result("old-4", "duplicate", score=0.95)], "u1")---def test_inject_tools_extract_query_and_has_tool_calls(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- monkeypatch.setattr(- handler,- "_get_memory_tools",- lambda: [- {- "function": {- "name": "memory_save",- "description": "save memory",- "parameters": {"type": "object"},- }- }- ],- )-- anthropic_tools, injected = handler.inject_tools([], "anthropic")- assert injected is True- assert anthropic_tools == [{"type": "memory_20250818", "name": "memory"}]-- custom_handler = MemoryHandler(- MemoryConfig(enabled=False, inject_tools=True), agent_type="codex"- )- monkeypatch.setattr(- custom_handler,- "_get_memory_tools",- lambda: [- {- "function": {- "name": "memory_save",- "description": "save memory",- "parameters": {"type": "object"},- }- }- ],- )- anthropic_custom, injected_custom = custom_handler.inject_tools([], "anthropic")- assert injected_custom is True- assert anthropic_custom == [- {"name": "memory_save", "description": "save memory", "input_schema": {"type": "object"}}- ]- openai_custom, _ = custom_handler.inject_tools([], "openai")- assert openai_custom == [- {- "function": {- "name": "memory_save",- "description": "save memory",- "parameters": {"type": "object"},- }- }- ]- existing, was_injected = custom_handler.inject_tools(- [{"function": {"name": "memory_save"}}],- "openai",- )- assert was_injected is False- assert existing == [{"function": {"name": "memory_save"}}]-- assert handler._extract_user_query([{"role": "assistant", "content": "skip"}]) == ""- assert handler._extract_user_query([{"role": "user", "content": "x" * 600}]) == "x" * 500- assert (- handler._extract_user_query(- [{"role": "user", "content": [{"type": "text", "text": "hello"}, {"type": "image"}]}]- )- == "hello"- )-- anthropic_response = {- "content": [{"type": "tool_use", "name": "memory_save", "id": "1", "input": {}}]- }- openai_response = {- "choices": [{"message": {"tool_calls": [{"id": "1", "function": {"name": "memory_save"}}]}}]- }- responses_api = {"output": [{"type": "function_call", "call_id": "2", "name": "memory"}]}- assert handler.has_memory_tool_calls(anthropic_response, "anthropic") is True- assert handler.has_memory_tool_calls(openai_response, "openai") is True- assert handler.has_memory_tool_calls(responses_api, "openai") is True- assert handler.has_memory_tool_calls({"content": []}, "anthropic") is False---async def test_memory_handler_misc_helpers(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:- handler = MemoryHandler(- MemoryConfig(- enabled=False,- use_native_tool=True,- inject_tools=True,- native_memory_dir=str(tmp_path / "native"),- ),- agent_type="codex",- )- first_lock = handler._get_init_lock()- assert handler._get_init_lock() is first_lock- assert handler.get_beta_headers() == {"anthropic-beta": "context-management-2025-06-27"}-- disabled_headers = MemoryHandler(- MemoryConfig(enabled=False, use_native_tool=False), agent_type="codex"- )- assert disabled_headers.get_beta_headers() == {}-- tools, injected = handler._inject_native_tool([])- assert injected is True- assert tools == [{"type": "memory_20250818", "name": "memory"}]- same_tools, same_injected = handler._inject_native_tool([{"name": "memory"}])- assert same_injected is False- assert same_tools == [{"name": "memory"}]-- calls = {"count": 0}- monkeypatch.setitem(- __import__("sys").modules,- "headroom.memory.tools",- SimpleNamespace(- get_memory_tools_optimized=lambda: calls.__setitem__("count", calls["count"] + 1)- or [{"name": "tool"}]- ),- )- cache_handler = MemoryHandler(MemoryConfig(enabled=False), agent_type="codex")- assert cache_handler._get_memory_tools() == [{"name": "tool"}]- assert cache_handler._get_memory_tools() == [{"name": "tool"}]- assert calls["count"] == 1-- assert cache_handler._extract_tool_calls(- {"content": [{"type": "tool_use", "id": "1"}]}, "anthropic"- ) == [{"type": "tool_use", "id": "1"}]- assert cache_handler._extract_tool_calls(- {"choices": [{"message": {"tool_calls": [{"id": "2"}]}}]},- "openai",- ) == [{"id": "2"}]- assert cache_handler._extract_tool_calls(- {"output": [{"type": "function_call", "call_id": "3"}]},- "openai",- ) == [{"type": "function_call", "call_id": "3"}]- assert cache_handler._extract_tool_calls({}, "other") == []-- closed: list[str] = []-- class Closable:- async def close(self) -> None:- closed.append("closed")-- await cache_handler._close_backend_instance(Closable(), reason="test")- assert closed == ["closed"]- await cache_handler._close_backend_instance(SimpleNamespace(), reason="test")-- class BrokenCloser:- def close(self) -> None:- raise RuntimeError("boom")-- await cache_handler._close_backend_instance(BrokenCloser(), reason="test")---async def test_search_and_format_context_and_handle_memory_tool_calls(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- backend = FakeBackend()- handler._backend = backend- handler._initialized = True- backend.search_results = [- make_result("m1", "Alice likes pizza", score=0.8, related_entities=["Alice", "pizza"]),- make_result("m2", "below threshold", score=0.2),- ]-- inject_none = await handler.search_and_format_context(- "u1",- [{"role": "assistant", "content": "skip"}],- )- assert inject_none is None-- context = await handler.search_and_format_context(- "u1",- [{"role": "user", "content": "What food does Alice like?"}],- )- assert "## Relevant Memories for This User" in context- assert "1. Alice likes pizza" in context- assert "(Related: Alice, pizza)" in context-- backend.raise_on = "search"- assert (- await handler.search_and_format_context("u1", [{"role": "user", "content": "Question"}])- is None- )- backend.raise_on = None-- async def fake_ensure_initialized() -> None:- return None-- async def fake_execute_memory_tool(tool_name, input_data, user_id, provider="anthropic"): # noqa: ANN001- return f"ran:{tool_name}:{user_id}:{provider}:{input_data}"-- async def fake_execute_native(input_data, user_id): # noqa: ANN001- return f"native:{user_id}:{input_data}"-- monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)- monkeypatch.setattr(handler, "_execute_memory_tool", fake_execute_memory_tool)- monkeypatch.setattr(handler, "_execute_native_memory_tool", fake_execute_native)-- anthropic_results = await handler.handle_memory_tool_calls(- {- "content": [- {"type": "tool_use", "name": "memory_save", "id": "a1", "input": {"content": "x"}},- {"type": "tool_use", "name": "memory", "id": "a2", "input": {"command": "view"}},- {"type": "tool_use", "name": "other", "id": "a3", "input": {}},- ]- },- "u1",- "anthropic",- )- assert anthropic_results == [- {- "type": "tool_result",- "tool_use_id": "a1",- "content": "ran:memory_save:u1:anthropic:{'content': 'x'}",- },- {"type": "tool_result", "tool_use_id": "a2", "content": "native:u1:{'command': 'view'}"},- ]-- openai_results = await handler.handle_memory_tool_calls(- {- "choices": [- {- "message": {- "tool_calls": [- {- "id": "o1",- "function": {- "name": "memory_search",- "arguments": '{"query":"pizza"}',- },- },- {"id": "o2", "function": {"name": "other", "arguments": "{}"}},- ]- }- }- ]- },- "u1",- "openai",- )- assert openai_results == [- {- "role": "tool",- "tool_call_id": "o1",- "content": "ran:memory_search:u1:openai:{'query': 'pizza'}",- }- ]-- handler._backend = None- skipped = await handler.handle_memory_tool_calls(- {- "choices": [- {- "message": {- "tool_calls": [- {- "id": "o3",- "function": {"name": "memory_delete", "arguments": "{}"},- }- ]- }- }- ]- },- "u1",- "openai",- )- assert skipped == []---async def test_ensure_initialized_timeout_and_cancellation(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path-) -> None:- handler = MemoryHandler(- MemoryConfig(- enabled=True, use_native_tool=True, native_memory_dir=str(tmp_path / "native")- ),- agent_type="codex",- )- closed: list[str] = []-- class ClosableBackend:- async def close(self) -> None:- closed.append("closed")-- async def fake_init_backend_locked() -> None:- handler._backend = ClosableBackend()-- monkeypatch.setattr(handler, "_init_backend_locked", fake_init_backend_locked)-- async def fake_wait_for_timeout(coro, timeout): # noqa: ANN001- await coro- raise asyncio.TimeoutError-- monkeypatch.setattr(memory_handler_module.asyncio, "wait_for", fake_wait_for_timeout)- await handler._ensure_initialized()- assert handler.backend is None- assert handler.initialized is False- assert closed == ["closed"]-- async def fake_wait_for_cancel(coro, timeout): # noqa: ANN001- await coro- raise asyncio.CancelledError-- monkeypatch.setattr(memory_handler_module.asyncio, "wait_for", fake_wait_for_cancel)- with pytest.raises(asyncio.CancelledError):- await handler._ensure_initialized()- assert handler.backend is None- assert handler.initialized is False- assert closed == ["closed", "closed"]---async def test_init_backend_locked_local_and_bridge_import(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path-) -> None:- handler = MemoryHandler(- MemoryConfig(- enabled=True,- backend="local",- db_path=str(tmp_path / "memory.db"),- bridge_enabled=True,- bridge_auto_import=True,- bridge_md_paths=[str(tmp_path / "notes.md")],- bridge_md_format="auto",- bridge_export_path=str(tmp_path / "export"),- ),- agent_type="codex",- )- seen: dict[str, object] = {}-- class FakeLocalBackendConfig:- def __init__(self, **kwargs): # noqa: ANN003- seen["config"] = kwargs- for key, value in kwargs.items():- setattr(self, key, value)-- class FakeLocalBackend:- def __init__(self, config) -> None: # noqa: ANN001- seen["backend_config"] = config-- async def _ensure_initialized(self) -> None:- seen["backend_initialized"] = True-- async def fake_init_and_import_bridge() -> None:- seen["bridge_called"] = True-- monkeypatch.setitem(- sys.modules,- "headroom.memory.backends.local",- SimpleNamespace(- LocalBackend=FakeLocalBackend,- LocalBackendConfig=FakeLocalBackendConfig,- ),- )- monkeypatch.setitem(sys.modules, "onnxruntime", SimpleNamespace())- monkeypatch.setattr(handler, "_init_and_import_bridge", fake_init_and_import_bridge)-- await handler._init_backend_locked()-- assert handler.initialized is True- assert seen["backend_initialized"] is True- assert seen["bridge_called"] is True- assert seen["config"] == {- "db_path": str(tmp_path / "memory.db"),- "embedder_backend": "onnx",- "embedder_model": "all-MiniLM-L6-v2",- "vector_dimension": 384,- }---async def test_init_and_import_bridge_success_and_failure(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path-) -> None:- handler = MemoryHandler(- MemoryConfig(- enabled=True,- bridge_enabled=True,- bridge_md_paths=[str(tmp_path / "notes.md")],- bridge_md_format="auto",- bridge_export_path=str(tmp_path / "export"),- ),- agent_type="codex",- )- handler._backend = object()- seen: dict[str, object] = {}-- class FakeMarkdownFormat(str):- pass-- class FakeBridgeConfig:- def __init__(self, **kwargs): # noqa: ANN003- seen["bridge_config"] = kwargs-- class FakeMemoryBridge:- def __init__(self, config, backend) -> None: # noqa: ANN001- seen["bridge_backend"] = backend- self.config = config-- async def import_from_markdown(self):- return SimpleNamespace(sections_imported=2, sections_skipped_duplicate=1)-- monkeypatch.setitem(- sys.modules,- "headroom.memory.bridge",- SimpleNamespace(MemoryBridge=FakeMemoryBridge),- )- monkeypatch.setitem(- sys.modules,- "headroom.memory.bridge_config",- SimpleNamespace(- BridgeConfig=FakeBridgeConfig,- MarkdownFormat=FakeMarkdownFormat,- ),- )-- await handler._init_and_import_bridge()- assert isinstance(handler._bridge, FakeMemoryBridge)- assert seen["bridge_backend"] is handler._backend- assert seen["bridge_config"] == {- "md_paths": [tmp_path / "notes.md"],- "md_format": "auto",- "auto_import_on_startup": True,- "export_path": tmp_path / "export",- }-- class BrokenMemoryBridge(FakeMemoryBridge):- async def import_from_markdown(self):- raise RuntimeError("bridge failed")-- handler._bridge = None- monkeypatch.setitem(- sys.modules,- "headroom.memory.bridge",- SimpleNamespace(MemoryBridge=BrokenMemoryBridge),- )- await handler._init_and_import_bridge()- assert isinstance(handler._bridge, BrokenMemoryBridge)---def test_memory_handler_init_defaults_and_tool_injection_edges(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path-) -> None:- native_dir = tmp_path / "workspace-memory"- monkeypatch.setattr(- "headroom.paths.native_memory_dir",- lambda: native_dir,- )- handler = MemoryHandler(MemoryConfig(enabled=False, use_native_tool=True), agent_type="codex")- assert handler._native_memory_dir == native_dir- assert native_dir.exists()-- disabled_injection = MemoryHandler(- MemoryConfig(enabled=False, inject_tools=False, use_native_tool=False),- agent_type="codex",- )- assert disabled_injection.inject_tools(None, "openai") == ([], False)-- same_type_tools, same_type_injected = handler._inject_native_tool(- [{"type": "memory_20250818", "name": "other"}]- )- assert same_type_injected is False- assert same_type_tools == [{"type": "memory_20250818", "name": "other"}]---async def test_ensure_initialized_fast_paths_and_qdrant_variants(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path-) -> None:- disabled = MemoryHandler(MemoryConfig(enabled=False), agent_type="codex")- await disabled._ensure_initialized()- assert disabled.initialized is False-- initialized = MemoryHandler(MemoryConfig(enabled=True), agent_type="codex")- initialized._initialized = True- await initialized._ensure_initialized()- assert initialized.initialized is True-- qdrant_handler = MemoryHandler(- MemoryConfig(enabled=True, backend="qdrant-neo4j"),- agent_type="codex",- )- seen: dict[str, object] = {}-- class FakeMem0Config:- def __init__(self, **kwargs): # noqa: ANN003- seen["config"] = kwargs-- class FakeAdapter:- def __init__(self, config) -> None: # noqa: ANN001- seen["adapter_config"] = config-- async def ensure_initialized(self) -> None:- seen["initialized"] = True-- monkeypatch.setitem(- sys.modules,- "headroom.memory.backends.direct_mem0",- SimpleNamespace(DirectMem0Adapter=FakeAdapter, Mem0Config=FakeMem0Config),- )- await qdrant_handler._init_backend_locked()- assert qdrant_handler.initialized is True- assert seen["initialized"] is True- assert seen["config"] == {- "qdrant_host": "localhost",- "qdrant_port": 6333,- "neo4j_uri": "neo4j://localhost:7687",- "neo4j_user": "neo4j",- "neo4j_password": "password",- "enable_graph": True,- }-- monkeypatch.setitem(sys.modules, "headroom.memory.backends.direct_mem0", None)- import builtins-- real_import = builtins.__import__-- def fake_import(name, globals=None, locals=None, fromlist=(), level=0):- if name == "headroom.memory.backends.direct_mem0":- raise ImportError("missing mem0")- return real_import(name, globals, locals, fromlist, level)-- monkeypatch.setattr(builtins, "__import__", fake_import)- broken_qdrant = MemoryHandler(- MemoryConfig(enabled=True, backend="qdrant-neo4j"),- agent_type="codex",- )- with pytest.raises(ImportError, match="missing mem0"):- await broken_qdrant._init_backend_locked()- monkeypatch.setattr(builtins, "__import__", real_import)-- unknown = MemoryHandler(MemoryConfig(enabled=True, backend="local"), agent_type="codex")- unknown.config.backend = "mystery" # type: ignore[assignment]- with pytest.raises(ValueError, match="Unknown memory backend"):- await unknown._init_backend_locked()---async def test_init_and_import_bridge_early_return_and_context_formatting_edges(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path-) -> None:- handler = MemoryHandler(- MemoryConfig(- enabled=True, bridge_enabled=True, bridge_md_paths=[str(tmp_path / "notes.md")]- ),- agent_type="codex",- )- handler._bridge = object()- await handler._init_and_import_bridge()- assert handler._bridge is not None-- handler.config.inject_context = False- assert (- await handler.search_and_format_context("u1", [{"role": "user", "content": "hello"}])- is None- )-- handler.config.inject_context = True- handler._backend = FakeBackend()- handler._initialized = True- handler._backend.search_results = [make_result("m1", "too low", score=0.1)]- assert (- await handler.search_and_format_context(- "u1", [{"role": "user", "content": [{"type": "image"}]}]- )- is None- )- assert (- await handler.search_and_format_context("u1", [{"role": "user", "content": "hello"}])- is None- )---async def test_extract_tool_calls_and_handle_tool_calls_parse_edges(- handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch-) -> None:- assert handler._extract_tool_calls({"content": "bad"}, "anthropic") == []- assert handler._extract_tool_calls({"choices": []}, "openai") == []- assert handler._extract_tool_calls({"output": "bad"}, "openai") == []-- backend = FakeBackend()- handler._backend = backend-- async def fake_ensure_initialized() -> None:- return None-- async def fake_execute(tool_name, input_data, user_id, provider="anthropic"): # noqa: ANN001- return f"ok:{tool_name}:{input_data}"-- monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)- monkeypatch.setattr(handler, "_execute_memory_tool", fake_execute)-- results = await handler.handle_memory_tool_calls(- {- "output": [- {- "type": "function_call",- "call_id": "fc1",- "name": "memory_search",- "arguments": "{bad",- },- {"type": "function_call", "call_id": "fc2", "name": "other", "arguments": "{}"},- ]- },- "u1",- "openai",- )- assert results == [{"role": "tool", "tool_call_id": "fc1", "content": "ok:memory_search:{}"}]+from __future__ import annotations++import asyncio+import json+import sys+from pathlib import Path+from types import SimpleNamespace++import pytest++from headroom.proxy import memory_handler as memory_handler_module+from headroom.proxy.memory_handler import MemoryConfig, MemoryHandler+++def handler(tmp_path: Path) -> MemoryHandler:+ return MemoryHandler(+ MemoryConfig(+ enabled=False,+ use_native_tool=True,+ native_memory_dir=str(tmp_path / "native"),+ ),+ agent_type="codex",+ )+++class FakeBackend:+ def __init__(self) -> None:+ self.search_results: list[object] = []+ self.saved: list[dict[str, object]] = []+ self.updated: list[dict[str, object]] = []+ self.deleted: list[str] = []+ self.raise_on: str | None = None++ async def search_memories(self, **kwargs): # noqa: ANN003+ if self.raise_on == "search":+ raise RuntimeError("search failed")+ return self.search_results++ async def save_memory(self, **kwargs): # noqa: ANN003+ if self.raise_on == "save":+ raise RuntimeError("save failed")+ self.saved.append(kwargs)+ return SimpleNamespace(id=f"mem-{len(self.saved)}", content=kwargs["content"])++ async def update_memory(self, **kwargs): # noqa: ANN003+ if self.raise_on == "update":+ raise RuntimeError("update failed")+ self.updated.append(kwargs)+ return SimpleNamespace(id=kwargs["memory_id"])++ async def delete_memory(self, memory_id: str): # noqa: ANN201+ if self.raise_on == "delete":+ raise RuntimeError("delete failed")+ self.deleted.append(memory_id)+ return True+++def make_result(+ memory_id: str,+ content: str,+ *,+ score: float = 0.9,+ metadata: dict[str, object] | None = None,+ related_entities: list[str] | None = None,+ created_at: str | None = None,+ importance: float = 0.5,+) -> object:+ return SimpleNamespace(+ memory=SimpleNamespace(+ id=memory_id,+ content=content,+ metadata=metadata or {},+ created_at=created_at,+ importance=importance,+ ),+ score=score,+ related_entities=related_entities or [],+ )+++def test_resolve_native_path_blocks_traversal(handler: MemoryHandler) -> None:+ resolved = handler._resolve_native_path("/memories/topic.txt", "u1")+ assert resolved.name == "topic.txt"+ assert "u1" in str(resolved)++ with pytest.raises(ValueError, match="Path traversal detected"):+ handler._resolve_native_path("/memories/../escape.txt", "u1")+++def test_native_view_lists_directory_and_reads_files(handler: MemoryHandler) -> None:+ root = handler._resolve_native_path("/memories", "u1")+ (root / "alpha.txt").write_text("line1\nline2\nline3", encoding="utf-8")+ (root / "nested").mkdir()+ (root / "nested" / "beta.txt").write_text("nested", encoding="utf-8")+ (root / ".hidden").write_text("skip", encoding="utf-8")+ (root / "node_modules").mkdir()++ listing = handler._native_view({"path": "/memories"}, "u1")+ assert "/memories/alpha.txt" in listing+ assert "/memories/nested/beta.txt" in listing+ assert ".hidden" not in listing+ assert "/memories/node_modules" not in listing++ file_view = handler._native_view({"path": "/memories/alpha.txt", "view_range": [2, 3]}, "u1")+ assert "2\tline2" in file_view+ assert "3\tline3" in file_view+++def test_native_view_handles_missing_paths_and_latin1(handler: MemoryHandler) -> None:+ missing = handler._native_view({"path": "/memories/missing.txt"}, "u1")+ assert "does not exist" in missing++ latin_path = handler._resolve_native_path("/memories/latin.txt", "u1")+ latin_path.write_bytes("caf\xe9".encode("latin-1"))+ viewed = handler._native_view({"path": "/memories/latin.txt"}, "u1")+ assert "cafe" not in viewed+ assert "café" in viewed+++def test_native_create_insert_delete_and_rename(handler: MemoryHandler) -> None:+ assert handler._native_create(+ {"path": "/memories/note.txt", "file_text": "a\nb"}, "u1"+ ).startswith("File created successfully")+ assert handler._native_create(+ {"path": "/memories/note.txt", "file_text": "dup"}, "u1"+ ).startswith("Error: File /memories/note.txt already exists")++ inserted = handler._native_insert(+ {"path": "/memories/note.txt", "insert_line": 1, "insert_text": "middle"},+ "u1",+ )+ assert inserted == "The file /memories/note.txt has been edited."+ assert "middle" in handler._resolve_native_path("/memories/note.txt", "u1").read_text(+ encoding="utf-8"+ )++ renamed = handler._native_rename(+ {"old_path": "/memories/note.txt", "new_path": "/memories/archive/renamed.txt"},+ "u1",+ )+ assert renamed == "Successfully renamed /memories/note.txt to /memories/archive/renamed.txt"++ deleted = handler._native_delete_file({"path": "/memories/archive"}, "u1")+ assert deleted == "Successfully deleted /memories/archive"+++def test_native_insert_validates_range_and_path(handler: MemoryHandler) -> None:+ assert (+ handler._native_insert({"insert_line": 0, "insert_text": "x"}, "u1")+ == "Error: path is required"+ )+ assert "does not exist" in handler._native_insert(+ {"path": "/memories/missing.txt", "insert_line": 0, "insert_text": "x"},+ "u1",+ )++ note = handler._resolve_native_path("/memories/note.txt", "u1")+ note.write_text("a\nb", encoding="utf-8")+ invalid = handler._native_insert(+ {"path": "/memories/note.txt", "insert_line": 4, "insert_text": "x"},+ "u1",+ )+ assert "Invalid `insert_line` parameter: 4" in invalid+++def test_native_str_replace_covers_missing_multiple_and_success(handler: MemoryHandler) -> None:+ note = handler._resolve_native_path("/memories/note.txt", "u1")+ note.write_text("hello\nhello\nworld", encoding="utf-8")++ assert (+ handler._native_str_replace({"old_str": "hello", "new_str": "bye"}, "u1")+ == "Error: path is required"+ )+ assert (+ handler._native_str_replace({"path": "/memories/note.txt", "new_str": "bye"}, "u1")+ == "Error: old_str is required"+ )++ multiple = handler._native_str_replace(+ {"path": "/memories/note.txt", "old_str": "hello", "new_str": "bye"},+ "u1",+ )+ assert "Multiple occurrences of old_str `hello` in lines: 1, 2" in multiple++ note.write_text("hello\nworld", encoding="utf-8")+ missing = handler._native_str_replace(+ {"path": "/memories/note.txt", "old_str": "nope", "new_str": "bye"},+ "u1",+ )+ assert "did not appear verbatim" in missing++ success = handler._native_str_replace(+ {"path": "/memories/note.txt", "old_str": "hello", "new_str": "bye"},+ "u1",+ )+ assert "The memory file has been edited." in success+ assert "bye" in note.read_text(encoding="utf-8")+++def test_native_delete_and_rename_validate_inputs(handler: MemoryHandler) -> None:+ assert handler._native_delete_file({}, "u1") == "Error: path is required"+ assert "does not exist" in handler._native_delete_file({"path": "/memories/missing.txt"}, "u1")++ assert (+ handler._native_rename({"new_path": "/memories/new.txt"}, "u1")+ == "Error: old_path is required"+ )+ assert (+ handler._native_rename({"old_path": "/memories/old.txt"}, "u1")+ == "Error: new_path is required"+ )+ assert "does not exist" in handler._native_rename(+ {"old_path": "/memories/old.txt", "new_path": "/memories/new.txt"},+ "u1",+ )++ old = handler._resolve_native_path("/memories/old.txt", "u1")+ new = handler._resolve_native_path("/memories/new.txt", "u1")+ old.write_text("x", encoding="utf-8")+ new.write_text("y", encoding="utf-8")+ assert (+ handler._native_rename(+ {"old_path": "/memories/old.txt", "new_path": "/memories/new.txt"},+ "u1",+ )+ == "Error: The destination /memories/new.txt already exists"+ )+++async def test_execute_native_memory_tool_dispatches_and_wraps_errors(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ handler._backend = object()+ called: list[tuple[str, dict[str, object], str]] = []++ async def fake_ensure_initialized() -> None:+ return None++ async def fake_view(input_data, user_id): # noqa: ANN001+ called.append(("view", input_data, user_id))+ return "viewed"++ async def fake_create(input_data, user_id): # noqa: ANN001+ called.append(("create", input_data, user_id))+ return "created"++ monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)+ monkeypatch.setattr(handler, "_native_view_semantic", fake_view)+ monkeypatch.setattr(handler, "_native_create_semantic", fake_create)++ assert await handler._execute_native_memory_tool({"command": "view"}, "u1") == "viewed"+ assert await handler._execute_native_memory_tool({"command": "create"}, "u1") == "created"+ assert (+ await handler._execute_native_memory_tool({"command": "bad"}, "u1")+ == "Error: Unknown command 'bad'"+ )++ async def boom(input_data, user_id): # noqa: ANN001+ raise RuntimeError("oops")++ monkeypatch.setattr(handler, "_native_view_semantic", boom)+ assert await handler._execute_native_memory_tool({"command": "view"}, "u1") == "Error: oops"+ assert [entry[0] for entry in called] == ["view", "create"]+++async def test_semantic_search_recent_all_and_overview(handler: MemoryHandler) -> None:+ backend = FakeBackend()+ handler._backend = backend+ backend.search_results = [+ make_result(+ "m1",+ "Alice likes pizza and pasta",+ score=0.91,+ related_entities=["Alice", "pizza"],+ created_at="2026-04-22",+ ),+ make_result("m2", "Bob prefers ramen", score=0.83),+ ]++ search_text = await handler._semantic_search("pizza", "u1")+ assert "Found 2 memories matching 'pizza'" in search_text+ assert "[91% match] Alice likes pizza and pasta" in search_text+ assert "Related: Alice, pizza" in search_text++ recent_text = await handler._get_recent_memories("u1", limit=2)+ assert "Recent memories:" in recent_text+ assert "(2026-04-22)" in recent_text++ all_text = await handler._list_all_memories("u1", limit=2)+ assert "Showing up to 2 memories:" in all_text+ assert "Showing first 2" in all_text++ overview = await handler._get_memory_overview("u1")+ assert "Memory System (2 memories stored)" in overview+ assert "view /memories/search/<your query>" in overview+++async def test_semantic_helpers_handle_empty_backend_and_errors(handler: MemoryHandler) -> None:+ assert await handler._semantic_search("x", "u1") == "Error: Memory backend not initialized"+ assert await handler._get_recent_memories("u1") == "Error: Memory backend not initialized"+ assert await handler._list_all_memories("u1") == "Error: Memory backend not initialized"+ assert await handler._get_memory_overview("u1") == "Error: Memory backend not initialized"++ backend = FakeBackend()+ handler._backend = backend+ assert "No memories found matching 'x'" in await handler._semantic_search("x", "u1")+ assert "No memories stored yet." in await handler._list_all_memories("u1")+ assert "No memories stored yet." in await handler._get_recent_memories("u1")++ backend.raise_on = "search"+ assert "Error searching memories: search failed" == await handler._semantic_search("x", "u1")+ assert "Error getting recent memories: search failed" == await handler._get_recent_memories(+ "u1"+ )+ assert "Error listing memories: search failed" == await handler._list_all_memories("u1")+ overview = await handler._get_memory_overview("u1")+ assert "📁 Memory System" in overview+ assert "To SEARCH memories" in overview+++async def test_native_view_semantic_routes_paths(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ seen: list[tuple[str, object]] = []++ async def fake_search(query, user_id, top_k=5): # noqa: ANN001+ seen.append(("search", query))+ return "search-result"++ async def fake_recent(user_id, limit=10): # noqa: ANN001+ seen.append(("recent", limit))+ return "recent-result"++ async def fake_all(user_id, limit=20): # noqa: ANN001+ seen.append(("all", limit))+ return "all-result"++ async def fake_overview(user_id): # noqa: ANN001+ seen.append(("overview", user_id))+ return "overview-result"++ monkeypatch.setattr(handler, "_semantic_search", fake_search)+ monkeypatch.setattr(handler, "_get_recent_memories", fake_recent)+ monkeypatch.setattr(handler, "_list_all_memories", fake_all)+ monkeypatch.setattr(handler, "_get_memory_overview", fake_overview)++ assert (+ await handler._native_view_semantic({"path": "/memories/search/pizza"}, "u1")+ == "search-result"+ )+ assert (+ await handler._native_view_semantic({"path": "/memories/recent"}, "u1") == "recent-result"+ )+ assert await handler._native_view_semantic({"path": "/memories/all"}, "u1") == "all-result"+ assert await handler._native_view_semantic({"path": "/memories"}, "u1") == "overview-result"+ assert (+ await handler._native_view_semantic({"path": "/memories/work/projects"}, "u1")+ == "search-result"+ )+ assert (await handler._native_view_semantic({"path": "/memories/search/"}, "u1")).startswith(+ "Error: Please provide a search query"+ )+ assert seen == [+ ("search", "pizza"),+ ("recent", 10),+ ("all", 20),+ ("overview", "u1"),+ ("search", "work projects"),+ ]+++async def test_native_semantic_create_append_delete_and_rename(handler: MemoryHandler) -> None:+ backend = FakeBackend()+ handler._backend = backend++ assert await handler._native_create_semantic({}, "u1") == "Error: path is required"+ assert (+ await handler._native_create_semantic({"path": "/memories/topic.txt"}, "u1")+ == "Error: file_text is required (the memory content)"+ )++ created = await handler._native_create_semantic(+ {"path": "/memories/topic.txt", "file_text": "prefers pizza"},+ "u1",+ )+ assert created == "File created successfully at: /memories/topic.txt"+ assert backend.saved[-1]["metadata"] == {+ "virtual_path": "/memories/topic.txt",+ "topic": "topic",+ }++ assert await handler._native_append_semantic({}, "u1") == "Error: path is required"+ assert (+ await handler._native_append_semantic({"path": "/memories/topic.txt"}, "u1")+ == "Error: insert_text is required"+ )+ appended = await handler._native_append_semantic(+ {"path": "/memories/topic.txt", "insert_text": "and pasta"},+ "u1",+ )+ assert appended == "The file /memories/topic.txt has been edited."+ assert backend.saved[-1]["metadata"]["appended"] is True++ backend.search_results = [+ make_result(+ "m1", "prefers pizza", metadata={"virtual_path": "/memories/topic.txt"}, score=0.6+ ),+ make_result("m2", "prefers pasta", metadata={}, score=0.91),+ ]+ deleted = await handler._native_delete_semantic({"path": "/memories/topic.txt"}, "u1")+ assert deleted == "Successfully deleted /memories/topic.txt"+ assert backend.deleted == ["m1", "m2"]++ backend.search_results = [+ make_result(+ "m3", "old content", metadata={"virtual_path": "/memories/old.txt"}, importance=0.7+ )+ ]+ renamed = await handler._native_rename_semantic(+ {"old_path": "/memories/old.txt", "new_path": "/memories/new/topic.txt"},+ "u1",+ )+ assert renamed == "Successfully renamed /memories/old.txt to /memories/new/topic.txt"+ assert backend.deleted[-1] == "m3"+ assert backend.saved[-1]["metadata"] == {+ "virtual_path": "/memories/new/topic.txt",+ "topic": "new_topic",+ }+++async def test_native_semantic_update_delete_rename_and_backend_errors(+ handler: MemoryHandler,+) -> None:+ backend = FakeBackend()+ handler._backend = backend++ assert await handler._native_update_semantic({}, "u1") == "Error: path is required"+ assert (+ await handler._native_update_semantic({"path": "/memories/t.txt"}, "u1")+ == "Error: old_str is required"+ )++ backend.search_results = [+ make_result("m1", "hello hello world", metadata={"virtual_path": "/memories/t.txt"})+ ]+ multi = await handler._native_update_semantic(+ {"path": "/memories/t.txt", "old_str": "hello", "new_str": "bye"},+ "u1",+ )+ assert "Multiple occurrences of old_str `hello`" in multi++ backend.search_results = [+ make_result("m1", "hello world", metadata={"virtual_path": "/memories/t.txt"})+ ]+ edited = await handler._native_update_semantic(+ {"path": "/memories/t.txt", "old_str": "hello", "new_str": "bye"},+ "u1",+ )+ assert "The memory file has been edited." in edited+ assert backend.updated[-1]["new_content"] == "bye world"++ class NoUpdateBackend:+ def __init__(self) -> None:+ self.search_results: list[object] = []+ self.saved: list[dict[str, object]] = []+ self.deleted: list[str] = []++ async def search_memories(self, **kwargs): # noqa: ANN003+ return self.search_results++ async def delete_memory(self, memory_id: str): # noqa: ANN201+ self.deleted.append(memory_id)+ return True++ async def save_memory(self, **kwargs): # noqa: ANN003+ self.saved.append(kwargs)+ return SimpleNamespace(id=f"mem-{len(self.saved)}")++ no_update_backend = NoUpdateBackend()+ no_update_backend.search_results = [+ make_result("m2", "alpha beta", metadata={"virtual_path": "/memories/t.txt"})+ ]+ handler._backend = no_update_backend+ fallback = await handler._native_update_semantic(+ {"path": "/memories/t.txt", "old_str": "alpha", "new_str": "omega"},+ "u1",+ )+ assert "The memory file has been edited." in fallback+ assert no_update_backend.deleted[-1] == "m2"+ assert no_update_backend.saved[-1]["content"] == "omega beta"++ backend = FakeBackend()+ handler._backend = backend+ assert await handler._native_delete_semantic({}, "u1") == "Error: path is required"+ assert await handler._native_rename_semantic({}, "u1") == "Error: old_path is required"+ assert (+ await handler._native_rename_semantic({"old_path": "/memories/a.txt"}, "u1")+ == "Error: new_path is required"+ )+ assert (+ await handler._native_delete_semantic({"path": "/memories/x.txt"}, "u1")+ == "Error: The path /memories/x.txt does not exist"+ )+ assert (+ await handler._native_rename_semantic(+ {"old_path": "/memories/x.txt", "new_path": "/memories/y.txt"},+ "u1",+ )+ == "Error: The path /memories/x.txt does not exist"+ )++ backend.search_results = [+ make_result("m9", "content", metadata={"virtual_path": "/memories/other.txt"}, score=0.1)+ ]+ assert (+ await handler._native_delete_semantic({"path": "/memories/x.txt"}, "u1")+ == "Error: The path /memories/x.txt does not exist"+ )+ assert (+ await handler._native_rename_semantic(+ {"old_path": "/memories/x.txt", "new_path": "/memories/y.txt"},+ "u1",+ )+ == "Error: The path /memories/x.txt does not exist"+ )++ backend.raise_on = "search"+ assert (+ await handler._native_create_semantic(+ {"path": "/memories/topic.txt", "file_text": "content"},+ "u1",+ )+ == "File created successfully at: /memories/topic.txt"+ )+ backend.raise_on = "save"+ assert (+ await handler._native_create_semantic(+ {"path": "/memories/topic.txt", "file_text": "content"}, "u1"+ )+ ).startswith("Error: ")+ assert (+ await handler._native_append_semantic(+ {"path": "/memories/topic.txt", "insert_text": "content"}, "u1"+ )+ ).startswith("Error: ")+ backend.raise_on = "search"+ assert (+ await handler._native_update_semantic(+ {"path": "/memories/t.txt", "old_str": "a", "new_str": "b"}, "u1"+ )+ ).startswith("Error: ")+ assert (await handler._native_delete_semantic({"path": "/memories/x.txt"}, "u1")).startswith(+ "Error: "+ )+ assert (+ await handler._native_rename_semantic(+ {"old_path": "/memories/x.txt", "new_path": "/memories/y.txt"},+ "u1",+ )+ ).startswith("Error: ")+++async def test_execute_search_update_delete_and_handler_status(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ backend = FakeBackend()+ handler._backend = backend++ assert json.loads(await handler._execute_search({}, "u1")) == {+ "status": "error",+ "error": "query is required",+ }++ backend.search_results = [+ make_result("m1", "pizza", score=0.9123, related_entities=["food", "italy"])+ ]+ search_payload = json.loads(+ await handler._execute_search(+ {"query": "pizza", "top_k": 3, "include_related": False, "entities": ["food"]},+ "u1",+ )+ )+ assert search_payload["status"] == "found"+ assert search_payload["count"] == 1+ assert search_payload["memories"][0] == {+ "id": "m1",+ "content": "pizza",+ "score": 0.912,+ "entities": ["food", "italy"],+ }++ assert json.loads(await handler._execute_update({}, "u1")) == {+ "status": "error",+ "error": "memory_id is required",+ }+ assert json.loads(await handler._execute_update({"memory_id": "m1"}, "u1")) == {+ "status": "error",+ "error": "new_content is required",+ }++ backend.search_results = [make_result("m1", "old content")]+ update_payload = json.loads(+ await handler._execute_update(+ {"memory_id": "m1", "new_content": "new content", "reason": "cleanup"},+ "u1",+ provider="openai",+ )+ )+ assert update_payload == {"status": "updated", "memory_id": "m1"}+ assert backend.updated[-1]["new_content"] == "new content"++ class NoUpdateBackend:+ def __init__(self) -> None:+ self.deleted: list[str] = []+ self.saved: list[dict[str, object]] = []++ async def delete_memory(self, memory_id: str): # noqa: ANN201+ self.deleted.append(memory_id)+ return True++ async def save_memory(self, **kwargs): # noqa: ANN003+ self.saved.append(kwargs)+ return SimpleNamespace(id="m2")++ no_update_backend = NoUpdateBackend()+ handler._backend = no_update_backend+ update_fallback = json.loads(+ await handler._execute_update({"memory_id": "m1", "new_content": "replacement"}, "u1")+ )+ assert update_fallback == {+ "status": "updated",+ "memory_id": "m2",+ "note": "Replaced via delete+save",+ }+ assert no_update_backend.deleted == ["m1"]++ handler._backend = backend+ assert json.loads(await handler._execute_delete({}, "u1")) == {+ "status": "error",+ "error": "memory_id is required",+ }+ delete_payload = json.loads(await handler._execute_delete({"memory_id": "m1"}, "u1"))+ assert delete_payload == {"status": "deleted", "memory_id": "m1"}++ assert handler.health_status() == {+ "enabled": False,+ "backend": "local",+ "initialized": False,+ "native_tool": True,+ "bridge_enabled": False,+ }++ seen = {"count": 0}++ async def fake_ensure_initialized() -> None:+ seen["count"] += 1++ monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)+ await handler.ensure_initialized()+ assert seen["count"] == 1+++async def test_warmup_embedder_and_close(handler: MemoryHandler) -> None:+ assert await handler.warmup_embedder() is False++ class FakeEmbedder:+ def __init__(self) -> None:+ self.calls: list[str] = []++ async def embed(self, value: str) -> None:+ self.calls.append(value)++ embedder = FakeEmbedder()+ handler._initialized = True+ handler._backend = SimpleNamespace(_hierarchical_memory=SimpleNamespace(_embedder=embedder))+ assert await handler.warmup_embedder() is True+ assert embedder.calls == ["warmup"]++ handler._backend = SimpleNamespace(+ _hierarchical_memory=SimpleNamespace(_embedder=SimpleNamespace())+ )+ assert await handler.warmup_embedder() is False++ handler._backend = SimpleNamespace(close=lambda: None)+ await handler.close()+ assert handler.backend is None+ assert handler.initialized is False+++async def test_execute_memory_tool_save_and_background_dedup(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ backend = FakeBackend()+ handler._backend = backend++ assert json.loads(await handler._execute_memory_tool("unknown", {}, "u1")) == {+ "error": "Unknown tool: unknown"+ }+ assert json.loads(await handler._execute_memory_tool("memory_save", {}, "u1")) == {+ "status": "error",+ "error": "content is required",+ }++ created_tasks: list[object] = []++ def fake_create_task(coro): # noqa: ANN001+ created_tasks.append(coro)+ coro.close()+ return SimpleNamespace()++ monkeypatch.setattr("headroom.proxy.memory_handler.asyncio.create_task", fake_create_task)+ backend.search_results = [+ make_result(+ "other",+ "Very similar memory content " * 5,+ score=0.8,+ metadata={"source_agent": "claude"},+ ),+ make_result("mem-1", "self result", score=0.99),+ ]++ saved = json.loads(+ await handler._execute_memory_tool(+ "memory_save",+ {+ "content": "Useful fact",+ "importance": 0.7,+ "facts": ["fact"],+ "entities": ["entity"],+ "extracted_entities": ["entity"],+ "relationships": ["rel"],+ "extracted_relationships": ["rel"],+ },+ "u1",+ provider="openai",+ )+ )+ assert saved["status"] == "saved"+ assert saved["memory_id"] == "mem-1"+ assert "Similar memory exists" in saved["note"]+ assert "saved by claude" in saved["note"]+ assert backend.saved[-1]["metadata"]["source_provider"] == "openai"+ assert len(created_tasks) == 1++ backend.raise_on = "save"+ errored = json.loads(await handler._execute_memory_tool("memory_save", {"content": "x"}, "u1"))+ assert errored == {"status": "error", "error": "save failed"}+++async def test_execute_save_handles_search_failure_and_background_dedup_filters(+ handler: MemoryHandler,+) -> None:+ backend = FakeBackend()+ handler._backend = backend++ backend.raise_on = "search"+ saved = json.loads(await handler._execute_save({"content": "Useful fact"}, "u1"))+ assert saved == {"status": "saved", "memory_id": "mem-1", "content": "Useful fact"}++ backend.raise_on = None+ similar = [+ make_result("mem-1", "same", score=0.99),+ make_result("old-1", "duplicate", score=0.95, metadata={}),+ make_result("old-2", "already handled", score=0.99, metadata={"superseded_by": "new"}),+ make_result("old-3", "too low", score=0.5, metadata={}),+ ]+ await handler._background_dedup("mem-1", similar, "u1")+ assert backend.deleted == ["old-1"]++ backend.raise_on = "delete"+ await handler._background_dedup("mem-1", [make_result("old-4", "duplicate", score=0.95)], "u1")+++def test_inject_tools_extract_query_and_has_tool_calls(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(+ handler,+ "_get_memory_tools",+ lambda: [+ {+ "function": {+ "name": "memory_save",+ "description": "save memory",+ "parameters": {"type": "object"},+ }+ }+ ],+ )++ anthropic_tools, injected = handler.inject_tools([], "anthropic")+ assert injected is True+ assert anthropic_tools == [{"type": "memory_20250818", "name": "memory"}]++ custom_handler = MemoryHandler(+ MemoryConfig(enabled=False, inject_tools=True), agent_type="codex"+ )+ monkeypatch.setattr(+ custom_handler,+ "_get_memory_tools",+ lambda: [+ {+ "function": {+ "name": "memory_save",+ "description": "save memory",+ "parameters": {"type": "object"},+ }+ }+ ],+ )+ anthropic_custom, injected_custom = custom_handler.inject_tools([], "anthropic")+ assert injected_custom is True+ assert anthropic_custom == [+ {"name": "memory_save", "description": "save memory", "input_schema": {"type": "object"}}+ ]+ openai_custom, _ = custom_handler.inject_tools([], "openai")+ assert openai_custom == [+ {+ "function": {+ "name": "memory_save",+ "description": "save memory",+ "parameters": {"type": "object"},+ }+ }+ ]+ existing, was_injected = custom_handler.inject_tools(+ [{"function": {"name": "memory_save"}}],+ "openai",+ )+ assert was_injected is False+ assert existing == [{"function": {"name": "memory_save"}}]++ assert handler._extract_user_query([{"role": "assistant", "content": "skip"}]) == ""+ assert handler._extract_user_query([{"role": "user", "content": "x" * 600}]) == "x" * 500+ assert (+ handler._extract_user_query(+ [{"role": "user", "content": [{"type": "text", "text": "hello"}, {"type": "image"}]}]+ )+ == "hello"+ )++ anthropic_response = {+ "content": [{"type": "tool_use", "name": "memory_save", "id": "1", "input": {}}]+ }+ openai_response = {+ "choices": [{"message": {"tool_calls": [{"id": "1", "function": {"name": "memory_save"}}]}}]+ }+ responses_api = {"output": [{"type": "function_call", "call_id": "2", "name": "memory"}]}+ assert handler.has_memory_tool_calls(anthropic_response, "anthropic") is True+ assert handler.has_memory_tool_calls(openai_response, "openai") is True+ assert handler.has_memory_tool_calls(responses_api, "openai") is True+ assert handler.has_memory_tool_calls({"content": []}, "anthropic") is False+++async def test_memory_handler_misc_helpers(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:+ handler = MemoryHandler(+ MemoryConfig(+ enabled=False,+ use_native_tool=True,+ inject_tools=True,+ native_memory_dir=str(tmp_path / "native"),+ ),+ agent_type="codex",+ )+ first_lock = handler._get_init_lock()+ assert handler._get_init_lock() is first_lock+ assert handler.get_beta_headers() == {"anthropic-beta": "context-management-2025-06-27"}++ disabled_headers = MemoryHandler(+ MemoryConfig(enabled=False, use_native_tool=False), agent_type="codex"+ )+ assert disabled_headers.get_beta_headers() == {}++ tools, injected = handler._inject_native_tool([])+ assert injected is True+ assert tools == [{"type": "memory_20250818", "name": "memory"}]+ same_tools, same_injected = handler._inject_native_tool([{"name": "memory"}])+ assert same_injected is False+ assert same_tools == [{"name": "memory"}]++ calls = {"count": 0}+ monkeypatch.setitem(+ __import__("sys").modules,+ "headroom.memory.tools",+ SimpleNamespace(+ get_memory_tools_optimized=lambda: calls.__setitem__("count", calls["count"] + 1)+ or [{"name": "tool"}]+ ),+ )+ cache_handler = MemoryHandler(MemoryConfig(enabled=False), agent_type="codex")+ assert cache_handler._get_memory_tools() == [{"name": "tool"}]+ assert cache_handler._get_memory_tools() == [{"name": "tool"}]+ assert calls["count"] == 1++ assert cache_handler._extract_tool_calls(+ {"content": [{"type": "tool_use", "id": "1"}]}, "anthropic"+ ) == [{"type": "tool_use", "id": "1"}]+ assert cache_handler._extract_tool_calls(+ {"choices": [{"message": {"tool_calls": [{"id": "2"}]}}]},+ "openai",+ ) == [{"id": "2"}]+ assert cache_handler._extract_tool_calls(+ {"output": [{"type": "function_call", "call_id": "3"}]},+ "openai",+ ) == [{"type": "function_call", "call_id": "3"}]+ assert cache_handler._extract_tool_calls({}, "other") == []++ closed: list[str] = []++ class Closable:+ async def close(self) -> None:+ closed.append("closed")++ await cache_handler._close_backend_instance(Closable(), reason="test")+ assert closed == ["closed"]+ await cache_handler._close_backend_instance(SimpleNamespace(), reason="test")++ class BrokenCloser:+ def close(self) -> None:+ raise RuntimeError("boom")++ await cache_handler._close_backend_instance(BrokenCloser(), reason="test")+++async def test_search_and_format_context_and_handle_memory_tool_calls(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ backend = FakeBackend()+ handler._backend = backend+ handler._initialized = True+ backend.search_results = [+ make_result("m1", "Alice likes pizza", score=0.8, related_entities=["Alice", "pizza"]),+ make_result("m2", "below threshold", score=0.2),+ ]++ inject_none = await handler.search_and_format_context(+ "u1",+ [{"role": "assistant", "content": "skip"}],+ )+ assert inject_none is None++ context = await handler.search_and_format_context(+ "u1",+ [{"role": "user", "content": "What food does Alice like?"}],+ )+ assert "## Relevant Memories for This User" in context+ assert "1. Alice likes pizza" in context+ assert "(Related: Alice, pizza)" in context++ backend.raise_on = "search"+ assert (+ await handler.search_and_format_context("u1", [{"role": "user", "content": "Question"}])+ is None+ )+ backend.raise_on = None++ async def fake_ensure_initialized() -> None:+ return None++ async def fake_execute_memory_tool(tool_name, input_data, user_id, provider="anthropic"): # noqa: ANN001+ return f"ran:{tool_name}:{user_id}:{provider}:{input_data}"++ async def fake_execute_native(input_data, user_id): # noqa: ANN001+ return f"native:{user_id}:{input_data}"++ monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)+ monkeypatch.setattr(handler, "_execute_memory_tool", fake_execute_memory_tool)+ monkeypatch.setattr(handler, "_execute_native_memory_tool", fake_execute_native)++ anthropic_results = await handler.handle_memory_tool_calls(+ {+ "content": [+ {"type": "tool_use", "name": "memory_save", "id": "a1", "input": {"content": "x"}},+ {"type": "tool_use", "name": "memory", "id": "a2", "input": {"command": "view"}},+ {"type": "tool_use", "name": "other", "id": "a3", "input": {}},+ ]+ },+ "u1",+ "anthropic",+ )+ assert anthropic_results == [+ {+ "type": "tool_result",+ "tool_use_id": "a1",+ "content": "ran:memory_save:u1:anthropic:{'content': 'x'}",+ },+ {"type": "tool_result", "tool_use_id": "a2", "content": "native:u1:{'command': 'view'}"},+ ]++ openai_results = await handler.handle_memory_tool_calls(+ {+ "choices": [+ {+ "message": {+ "tool_calls": [+ {+ "id": "o1",+ "function": {+ "name": "memory_search",+ "arguments": '{"query":"pizza"}',+ },+ },+ {"id": "o2", "function": {"name": "other", "arguments": "{}"}},+ ]+ }+ }+ ]+ },+ "u1",+ "openai",+ )+ assert openai_results == [+ {+ "role": "tool",+ "tool_call_id": "o1",+ "content": "ran:memory_search:u1:openai:{'query': 'pizza'}",+ }+ ]++ handler._backend = None+ skipped = await handler.handle_memory_tool_calls(+ {+ "choices": [+ {+ "message": {+ "tool_calls": [+ {+ "id": "o3",+ "function": {"name": "memory_delete", "arguments": "{}"},+ }+ ]+ }+ }+ ]+ },+ "u1",+ "openai",+ )+ assert skipped == []+++async def test_ensure_initialized_timeout_and_cancellation(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ handler = MemoryHandler(+ MemoryConfig(+ enabled=True, use_native_tool=True, native_memory_dir=str(tmp_path / "native")+ ),+ agent_type="codex",+ )+ closed: list[str] = []++ class ClosableBackend:+ async def close(self) -> None:+ closed.append("closed")++ async def fake_init_backend_locked() -> None:+ handler._backend = ClosableBackend()++ monkeypatch.setattr(handler, "_init_backend_locked", fake_init_backend_locked)++ async def fake_wait_for_timeout(coro, timeout): # noqa: ANN001+ await coro+ raise asyncio.TimeoutError++ monkeypatch.setattr(memory_handler_module.asyncio, "wait_for", fake_wait_for_timeout)+ await handler._ensure_initialized()+ assert handler.backend is None+ assert handler.initialized is False+ assert closed == ["closed"]++ async def fake_wait_for_cancel(coro, timeout): # noqa: ANN001+ await coro+ raise asyncio.CancelledError++ monkeypatch.setattr(memory_handler_module.asyncio, "wait_for", fake_wait_for_cancel)+ with pytest.raises(asyncio.CancelledError):+ await handler._ensure_initialized()+ assert handler.backend is None+ assert handler.initialized is False+ assert closed == ["closed", "closed"]+++async def test_init_backend_locked_local_and_bridge_import(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ handler = MemoryHandler(+ MemoryConfig(+ enabled=True,+ backend="local",+ db_path=str(tmp_path / "memory.db"),+ bridge_enabled=True,+ bridge_auto_import=True,+ bridge_md_paths=[str(tmp_path / "notes.md")],+ bridge_md_format="auto",+ bridge_export_path=str(tmp_path / "export"),+ ),+ agent_type="codex",+ )+ seen: dict[str, object] = {}++ class FakeLocalBackendConfig:+ def __init__(self, **kwargs): # noqa: ANN003+ seen["config"] = kwargs+ for key, value in kwargs.items():+ setattr(self, key, value)++ class FakeLocalBackend:+ def __init__(self, config) -> None: # noqa: ANN001+ seen["backend_config"] = config++ async def _ensure_initialized(self) -> None:+ seen["backend_initialized"] = True++ async def fake_init_and_import_bridge() -> None:+ seen["bridge_called"] = True++ monkeypatch.setitem(+ sys.modules,+ "headroom.memory.backends.local",+ SimpleNamespace(+ LocalBackend=FakeLocalBackend,+ LocalBackendConfig=FakeLocalBackendConfig,+ ),+ )+ monkeypatch.setitem(sys.modules, "onnxruntime", SimpleNamespace())+ monkeypatch.setattr(handler, "_init_and_import_bridge", fake_init_and_import_bridge)++ await handler._init_backend_locked()++ assert handler.initialized is True+ assert seen["backend_initialized"] is True+ assert seen["bridge_called"] is True+ assert seen["config"] == {+ "db_path": str(tmp_path / "memory.db"),+ "embedder_backend": "onnx",+ "embedder_model": "all-MiniLM-L6-v2",+ "vector_dimension": 384,+ }+++async def test_init_and_import_bridge_success_and_failure(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ handler = MemoryHandler(+ MemoryConfig(+ enabled=True,+ bridge_enabled=True,+ bridge_md_paths=[str(tmp_path / "notes.md")],+ bridge_md_format="auto",+ bridge_export_path=str(tmp_path / "export"),+ ),+ agent_type="codex",+ )+ handler._backend = object()+ seen: dict[str, object] = {}++ class FakeMarkdownFormat(str):+ pass++ class FakeBridgeConfig:+ def __init__(self, **kwargs): # noqa: ANN003+ seen["bridge_config"] = kwargs++ class FakeMemoryBridge:+ def __init__(self, config, backend) -> None: # noqa: ANN001+ seen["bridge_backend"] = backend+ self.config = config++ async def import_from_markdown(self):+ return SimpleNamespace(sections_imported=2, sections_skipped_duplicate=1)++ monkeypatch.setitem(+ sys.modules,+ "headroom.memory.bridge",+ SimpleNamespace(MemoryBridge=FakeMemoryBridge),+ )+ monkeypatch.setitem(+ sys.modules,+ "headroom.memory.bridge_config",+ SimpleNamespace(+ BridgeConfig=FakeBridgeConfig,+ MarkdownFormat=FakeMarkdownFormat,+ ),+ )++ await handler._init_and_import_bridge()+ assert isinstance(handler._bridge, FakeMemoryBridge)+ assert seen["bridge_backend"] is handler._backend+ assert seen["bridge_config"] == {+ "md_paths": [tmp_path / "notes.md"],+ "md_format": "auto",+ "auto_import_on_startup": True,+ "export_path": tmp_path / "export",+ }++ class BrokenMemoryBridge(FakeMemoryBridge):+ async def import_from_markdown(self):+ raise RuntimeError("bridge failed")++ handler._bridge = None+ monkeypatch.setitem(+ sys.modules,+ "headroom.memory.bridge",+ SimpleNamespace(MemoryBridge=BrokenMemoryBridge),+ )+ await handler._init_and_import_bridge()+ assert isinstance(handler._bridge, BrokenMemoryBridge)+++def test_memory_handler_init_defaults_and_tool_injection_edges(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ native_dir = tmp_path / "workspace-memory"+ monkeypatch.setattr(+ "headroom.paths.native_memory_dir",+ lambda: native_dir,+ )+ handler = MemoryHandler(MemoryConfig(enabled=False, use_native_tool=True), agent_type="codex")+ assert handler._native_memory_dir == native_dir+ assert native_dir.exists()++ disabled_injection = MemoryHandler(+ MemoryConfig(enabled=False, inject_tools=False, use_native_tool=False),+ agent_type="codex",+ )+ assert disabled_injection.inject_tools(None, "openai") == ([], False)++ same_type_tools, same_type_injected = handler._inject_native_tool(+ [{"type": "memory_20250818", "name": "other"}]+ )+ assert same_type_injected is False+ assert same_type_tools == [{"type": "memory_20250818", "name": "other"}]+++async def test_ensure_initialized_fast_paths_and_qdrant_variants(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ disabled = MemoryHandler(MemoryConfig(enabled=False), agent_type="codex")+ await disabled._ensure_initialized()+ assert disabled.initialized is False++ initialized = MemoryHandler(MemoryConfig(enabled=True), agent_type="codex")+ initialized._initialized = True+ await initialized._ensure_initialized()+ assert initialized.initialized is True++ qdrant_handler = MemoryHandler(+ MemoryConfig(enabled=True, backend="qdrant-neo4j"),+ agent_type="codex",+ )+ seen: dict[str, object] = {}++ class FakeMem0Config:+ def __init__(self, **kwargs): # noqa: ANN003+ seen["config"] = kwargs++ class FakeAdapter:+ def __init__(self, config) -> None: # noqa: ANN001+ seen["adapter_config"] = config++ async def ensure_initialized(self) -> None:+ seen["initialized"] = True++ monkeypatch.setitem(+ sys.modules,+ "headroom.memory.backends.direct_mem0",+ SimpleNamespace(DirectMem0Adapter=FakeAdapter, Mem0Config=FakeMem0Config),+ )+ await qdrant_handler._init_backend_locked()+ assert qdrant_handler.initialized is True+ assert seen["initialized"] is True+ assert seen["config"] == {+ "qdrant_host": "localhost",+ "qdrant_port": 6333,+ "neo4j_uri": "neo4j://localhost:7687",+ "neo4j_user": "neo4j",+ "neo4j_password": "password",+ "enable_graph": True,+ }++ monkeypatch.setitem(sys.modules, "headroom.memory.backends.direct_mem0", None)+ import builtins++ real_import = builtins.__import__++ def fake_import(name, globals=None, locals=None, fromlist=(), level=0):+ if name == "headroom.memory.backends.direct_mem0":+ raise ImportError("missing mem0")+ return real_import(name, globals, locals, fromlist, level)++ monkeypatch.setattr(builtins, "__import__", fake_import)+ broken_qdrant = MemoryHandler(+ MemoryConfig(enabled=True, backend="qdrant-neo4j"),+ agent_type="codex",+ )+ with pytest.raises(ImportError, match="missing mem0"):+ await broken_qdrant._init_backend_locked()+ monkeypatch.setattr(builtins, "__import__", real_import)++ unknown = MemoryHandler(MemoryConfig(enabled=True, backend="local"), agent_type="codex")+ unknown.config.backend = "mystery" # type: ignore[assignment]+ with pytest.raises(ValueError, match="Unknown memory backend"):+ await unknown._init_backend_locked()+++async def test_init_and_import_bridge_early_return_and_context_formatting_edges(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ handler = MemoryHandler(+ MemoryConfig(+ enabled=True, bridge_enabled=True, bridge_md_paths=[str(tmp_path / "notes.md")]+ ),+ agent_type="codex",+ )+ handler._bridge = object()+ await handler._init_and_import_bridge()+ assert handler._bridge is not None++ handler.config.inject_context = False+ assert (+ await handler.search_and_format_context("u1", [{"role": "user", "content": "hello"}])+ is None+ )++ handler.config.inject_context = True+ handler._backend = FakeBackend()+ handler._initialized = True+ handler._backend.search_results = [make_result("m1", "too low", score=0.1)]+ assert (+ await handler.search_and_format_context(+ "u1", [{"role": "user", "content": [{"type": "image"}]}]+ )+ is None+ )+ assert (+ await handler.search_and_format_context("u1", [{"role": "user", "content": "hello"}])+ is None+ )+++async def test_extract_tool_calls_and_handle_tool_calls_parse_edges(+ handler: MemoryHandler, monkeypatch: pytest.MonkeyPatch+) -> None:+ assert handler._extract_tool_calls({"content": "bad"}, "anthropic") == []+ assert handler._extract_tool_calls({"choices": []}, "openai") == []+ assert handler._extract_tool_calls({"output": "bad"}, "openai") == []++ backend = FakeBackend()+ handler._backend = backend++ async def fake_ensure_initialized() -> None:+ return None++ async def fake_execute(tool_name, input_data, user_id, provider="anthropic"): # noqa: ANN001+ return f"ok:{tool_name}:{input_data}"++ monkeypatch.setattr(handler, "_ensure_initialized", fake_ensure_initialized)+ monkeypatch.setattr(handler, "_execute_memory_tool", fake_execute)++ results = await handler.handle_memory_tool_calls(+ {+ "output": [+ {+ "type": "function_call",+ "call_id": "fc1",+ "name": "memory_search",+ "arguments": "{bad",+ },+ {"type": "function_call", "call_id": "fc2", "name": "other", "arguments": "{}"},+ ]+ },+ "u1",+ "openai",+ )+ assert results == [{"role": "tool", "tool_call_id": "fc1", "content": "ok:memory_search:{}"}]diff --git a/tests/test_memory_wrapper.py b/tests/test_memory_wrapper.pyindex 8f2bb41a8..ab9631023 100644--- a/tests/test_memory_wrapper.py+++ b/tests/test_memory_wrapper.py@@ -1,206 +1,206 @@-from __future__ import annotations--from pathlib import Path-from types import SimpleNamespace--import pytest--from headroom.memory.config import EmbedderBackend-from headroom.memory.wrapper import MemoryWrapper, _MemoryAPI, with_memory---class FakeMemory:- def __init__(self) -> None:- self.search_results: list[object] = []- self.add_calls: list[dict[str, object]] = []- self.query_results: list[object] = []- self.clear_result = 0-- async def search(self, **kwargs): # noqa: ANN003- self.last_search = kwargs- return self.search_results-- async def add(self, **kwargs): # noqa: ANN003- self.add_calls.append(kwargs)- return SimpleNamespace(id=f"mem-{len(self.add_calls)}", **kwargs)-- async def query(self, filter_value): # noqa: ANN001, ANN201- self.last_filter = filter_value- return self.query_results-- async def clear_scope(self, **kwargs): # noqa: ANN003- self.last_clear = kwargs- return self.clear_result---def make_client(content: str = "raw response") -> tuple[object, object]:- response = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=content))])-- def create(**kwargs): # noqa: ANN003, ANN202- create.kwargs = kwargs- return response-- client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create)))- return client, response---def test_memory_wrapper_lazy_initialization_and_factory(monkeypatch: pytest.MonkeyPatch) -> None:- client, _response = make_client()- fake_memory = FakeMemory()- seen: dict[str, object] = {}-- async def fake_create(config): # noqa: ANN001- seen["config"] = config- return fake_memory-- monkeypatch.setattr("headroom.memory.wrapper.HierarchicalMemory.create", fake_create)-- wrapper = MemoryWrapper(- client,- user_id="alice",- db_path="memory.db",- top_k=7,- session_id="session-1",- agent_id="agent-1",- embedder_backend=EmbedderBackend.OPENAI,- openai_api_key="sk-test",- )-- assert wrapper.chat.completions._wrapper is wrapper- assert wrapper._initialized is False-- api = wrapper.memory- assert isinstance(api, _MemoryAPI)- assert wrapper._initialized is True- assert wrapper._memory is fake_memory- assert seen["config"].db_path == Path("memory.db")- assert seen["config"].embedder_backend == EmbedderBackend.OPENAI- assert seen["config"].openai_api_key == "sk-test"-- wrapped = with_memory(client, user_id="bob", session_id="s2", agent_id="a2", top_k=3)- assert isinstance(wrapped, MemoryWrapper)- assert wrapped._client is client- assert wrapped._user_id == "bob"- assert wrapped._session_id == "s2"- assert wrapped._agent_id == "a2"- assert wrapped._top_k == 3---def test_inject_memories_handles_empty_and_inserts_context() -> None:- client, _response = make_client()- fake_memory = FakeMemory()- wrapper = MemoryWrapper(client, user_id="alice", _memory=fake_memory)-- no_user = [{"role": "assistant", "content": "skip"}]- assert wrapper._inject_memories(no_user) == no_user-- messages = [{"role": "user", "content": "Question?"}]- assert wrapper._inject_memories(messages) == messages-- fake_memory.search_results = [- SimpleNamespace(memory=SimpleNamespace(content="Prefers Python")),- SimpleNamespace(memory=SimpleNamespace(content="Works on APIs")),- ]- original = [- {"role": "system", "content": "System"},- {"role": "user", "content": "Question?"},- {"role": "user", "content": "Follow-up"},- ]- injected = wrapper._inject_memories(original)-- assert original[1]["content"] == "Question?"- assert injected[1]["content"].startswith(- "<context>\n- Prefers Python\n- Works on APIs\n</context>\n\n"- )- assert injected[2]["content"] == "Follow-up"- assert fake_memory.last_search == {- "query": "Follow-up",- "user_id": "alice",- "session_id": None,- "top_k": 5,- }---def test_store_memories_persists_only_nonempty_content() -> None:- client, _response = make_client()- fake_memory = FakeMemory()- wrapper = MemoryWrapper(- client,- user_id="alice",- session_id="session-1",- agent_id="agent-1",- _memory=fake_memory,- )-- wrapper._store_memories([{"content": "Remember this"}, {"content": ""}, {}])-- assert fake_memory.add_calls == [- {- "content": "Remember this",- "user_id": "alice",- "session_id": "session-1",- "agent_id": "agent-1",- "importance": 0.7,- }- ]---def test_wrapped_completions_create_injects_parses_and_stores(- monkeypatch: pytest.MonkeyPatch,-) -> None:- client, response = make_client("raw completion")- wrapper = MemoryWrapper(client, user_id="alice", _memory=FakeMemory())- stored: list[list[dict[str, str]]] = []-- monkeypatch.setattr(- wrapper,- "_inject_memories",- lambda messages: [{"role": "user", "content": "enhanced"}],- )- monkeypatch.setattr(- "headroom.memory.wrapper.inject_memory_instruction",- lambda messages, short=True: messages- + [{"role": "system", "content": "memory-instruction"}],- )- monkeypatch.setattr(- "headroom.memory.wrapper.parse_response_with_memory",- lambda content: SimpleNamespace(- content="clean response",- memories=[{"content": "saved memory"}],- ),- )- monkeypatch.setattr(wrapper, "_store_memories", lambda memories: stored.append(memories))-- result = wrapper.chat.completions.create(- messages=[{"role": "user", "content": "hello"}], model="x"- )-- assert result is response- assert response.choices[0].message.content == "clean response"- assert client.chat.completions.create.kwargs["messages"] == [- {"role": "user", "content": "enhanced"},- {"role": "system", "content": "memory-instruction"},- ]- assert stored == [[{"content": "saved memory"}]]---def test_memory_api_methods_delegate_to_underlying_memory() -> None:- fake_memory = FakeMemory()- memory_one = SimpleNamespace(id="m1", content="alpha")- memory_two = SimpleNamespace(id="m2", content="beta")- fake_memory.search_results = [- SimpleNamespace(memory=memory_one),- SimpleNamespace(memory=memory_two),- ]- fake_memory.query_results = [memory_one, memory_two]- fake_memory.clear_result = 2-- api = _MemoryAPI(fake_memory, user_id="alice", session_id="session-1", agent_id="agent-1")-- assert api.search("alpha", top_k=3) == [memory_one, memory_two]- added = api.add("new memory", importance=0.9)- assert added.content == "new memory"- assert api.get_all() == [memory_one, memory_two]- assert api.clear() == 2- assert api.stats() == {"total": 2}- assert fake_memory.last_clear == {"user_id": "alice"}+from __future__ import annotations++from pathlib import Path+from types import SimpleNamespace++import pytest++from headroom.memory.config import EmbedderBackend+from headroom.memory.wrapper import MemoryWrapper, _MemoryAPI, with_memory+++class FakeMemory:+ def __init__(self) -> None:+ self.search_results: list[object] = []+ self.add_calls: list[dict[str, object]] = []+ self.query_results: list[object] = []+ self.clear_result = 0++ async def search(self, **kwargs): # noqa: ANN003+ self.last_search = kwargs+ return self.search_results++ async def add(self, **kwargs): # noqa: ANN003+ self.add_calls.append(kwargs)+ return SimpleNamespace(id=f"mem-{len(self.add_calls)}", **kwargs)++ async def query(self, filter_value): # noqa: ANN001, ANN201+ self.last_filter = filter_value+ return self.query_results++ async def clear_scope(self, **kwargs): # noqa: ANN003+ self.last_clear = kwargs+ return self.clear_result+++def make_client(content: str = "raw response") -> tuple[object, object]:+ response = SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(content=content))])++ def create(**kwargs): # noqa: ANN003, ANN202+ create.kwargs = kwargs+ return response++ client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create)))+ return client, response+++def test_memory_wrapper_lazy_initialization_and_factory(monkeypatch: pytest.MonkeyPatch) -> None:+ client, _response = make_client()+ fake_memory = FakeMemory()+ seen: dict[str, object] = {}++ async def fake_create(config): # noqa: ANN001+ seen["config"] = config+ return fake_memory++ monkeypatch.setattr("headroom.memory.wrapper.HierarchicalMemory.create", fake_create)++ wrapper = MemoryWrapper(+ client,+ user_id="alice",+ db_path="memory.db",+ top_k=7,+ session_id="session-1",+ agent_id="agent-1",+ embedder_backend=EmbedderBackend.OPENAI,+ openai_api_key="sk-test",+ )++ assert wrapper.chat.completions._wrapper is wrapper+ assert wrapper._initialized is False++ api = wrapper.memory+ assert isinstance(api, _MemoryAPI)+ assert wrapper._initialized is True+ assert wrapper._memory is fake_memory+ assert seen["config"].db_path == Path("memory.db")+ assert seen["config"].embedder_backend == EmbedderBackend.OPENAI+ assert seen["config"].openai_api_key == "sk-test"++ wrapped = with_memory(client, user_id="bob", session_id="s2", agent_id="a2", top_k=3)+ assert isinstance(wrapped, MemoryWrapper)+ assert wrapped._client is client+ assert wrapped._user_id == "bob"+ assert wrapped._session_id == "s2"+ assert wrapped._agent_id == "a2"+ assert wrapped._top_k == 3+++def test_inject_memories_handles_empty_and_inserts_context() -> None:+ client, _response = make_client()+ fake_memory = FakeMemory()+ wrapper = MemoryWrapper(client, user_id="alice", _memory=fake_memory)++ no_user = [{"role": "assistant", "content": "skip"}]+ assert wrapper._inject_memories(no_user) == no_user++ messages = [{"role": "user", "content": "Question?"}]+ assert wrapper._inject_memories(messages) == messages++ fake_memory.search_results = [+ SimpleNamespace(memory=SimpleNamespace(content="Prefers Python")),+ SimpleNamespace(memory=SimpleNamespace(content="Works on APIs")),+ ]+ original = [+ {"role": "system", "content": "System"},+ {"role": "user", "content": "Question?"},+ {"role": "user", "content": "Follow-up"},+ ]+ injected = wrapper._inject_memories(original)++ assert original[1]["content"] == "Question?"+ assert injected[1]["content"].startswith(+ "<context>\n- Prefers Python\n- Works on APIs\n</context>\n\n"+ )+ assert injected[2]["content"] == "Follow-up"+ assert fake_memory.last_search == {+ "query": "Follow-up",+ "user_id": "alice",+ "session_id": None,+ "top_k": 5,+ }+++def test_store_memories_persists_only_nonempty_content() -> None:+ client, _response = make_client()+ fake_memory = FakeMemory()+ wrapper = MemoryWrapper(+ client,+ user_id="alice",+ session_id="session-1",+ agent_id="agent-1",+ _memory=fake_memory,+ )++ wrapper._store_memories([{"content": "Remember this"}, {"content": ""}, {}])++ assert fake_memory.add_calls == [+ {+ "content": "Remember this",+ "user_id": "alice",+ "session_id": "session-1",+ "agent_id": "agent-1",+ "importance": 0.7,+ }+ ]+++def test_wrapped_completions_create_injects_parses_and_stores(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ client, response = make_client("raw completion")+ wrapper = MemoryWrapper(client, user_id="alice", _memory=FakeMemory())+ stored: list[list[dict[str, str]]] = []++ monkeypatch.setattr(+ wrapper,+ "_inject_memories",+ lambda messages: [{"role": "user", "content": "enhanced"}],+ )+ monkeypatch.setattr(+ "headroom.memory.wrapper.inject_memory_instruction",+ lambda messages, short=True: messages+ + [{"role": "system", "content": "memory-instruction"}],+ )+ monkeypatch.setattr(+ "headroom.memory.wrapper.parse_response_with_memory",+ lambda content: SimpleNamespace(+ content="clean response",+ memories=[{"content": "saved memory"}],+ ),+ )+ monkeypatch.setattr(wrapper, "_store_memories", lambda memories: stored.append(memories))++ result = wrapper.chat.completions.create(+ messages=[{"role": "user", "content": "hello"}], model="x"+ )++ assert result is response+ assert response.choices[0].message.content == "clean response"+ assert client.chat.completions.create.kwargs["messages"] == [+ {"role": "user", "content": "enhanced"},+ {"role": "system", "content": "memory-instruction"},+ ]+ assert stored == [[{"content": "saved memory"}]]+++def test_memory_api_methods_delegate_to_underlying_memory() -> None:+ fake_memory = FakeMemory()+ memory_one = SimpleNamespace(id="m1", content="alpha")+ memory_two = SimpleNamespace(id="m2", content="beta")+ fake_memory.search_results = [+ SimpleNamespace(memory=memory_one),+ SimpleNamespace(memory=memory_two),+ ]+ fake_memory.query_results = [memory_one, memory_two]+ fake_memory.clear_result = 2++ api = _MemoryAPI(fake_memory, user_id="alice", session_id="session-1", agent_id="agent-1")++ assert api.search("alpha", top_k=3) == [memory_one, memory_two]+ added = api.add("new memory", importance=0.9)+ assert added.content == "new memory"+ assert api.get_all() == [memory_one, memory_two]+ assert api.clear() == 2+ assert api.stats() == {"total": 2}+ assert fake_memory.last_clear == {"user_id": "alice"}diff --git a/tests/test_pipeline.py b/tests/test_pipeline.pyindex f6229a02e..ce7d1b78a 100644--- a/tests/test_pipeline.py+++ b/tests/test_pipeline.py@@ -1,130 +1,130 @@-from __future__ import annotations--import importlib.metadata-from dataclasses import dataclass--from headroom.pipeline import (- CANONICAL_PIPELINE_STAGES,- ENTRY_POINT_GROUP,- PipelineEvent,- PipelineExtensionManager,- PipelineStage,- discover_pipeline_extensions,- summarize_routing_markers,-)---@dataclass-class FakeEntryPoint:- name: str- value: object-- def load(self):- if isinstance(self.value, Exception):- raise self.value- return self.value---def test_discover_pipeline_extensions_handles_load_and_init_failures(- monkeypatch,-) -> None:- class WorkingExtension:- def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201- return event-- class NeedsInit:- def __init__(self) -> None:- raise RuntimeError("bad init")-- monkeypatch.setattr(- importlib.metadata,- "entry_points",- lambda group=None: [- FakeEntryPoint("working-instance", WorkingExtension()),- FakeEntryPoint("working-class", WorkingExtension),- FakeEntryPoint("bad-load", RuntimeError("bad load")),- FakeEntryPoint("bad-init", NeedsInit),- ]- if group == ENTRY_POINT_GROUP- else [],- )-- discovered = discover_pipeline_extensions()- assert len(discovered) == 2- assert all(callable(getattr(ext, "on_pipeline_event", None)) for ext in discovered)---def test_discover_pipeline_extensions_handles_enumeration_failure(monkeypatch) -> None:- monkeypatch.setattr(- importlib.metadata,- "entry_points",- lambda group=None: (_ for _ in ()).throw(RuntimeError("boom")),- )- assert discover_pipeline_extensions() == []---def test_pipeline_manager_emit_and_summary(monkeypatch) -> None:- class Hook:- def __init__(self) -> None:- self.seen: list[str] = []-- def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201- self.seen.append(event.stage.value)- event.metadata["hook"] = True- return event-- class ReplacingExtension:- def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201- return PipelineEvent(- stage=event.stage,- operation=event.operation,- request_id=event.request_id,- provider=event.provider,- model=event.model,- messages=event.messages,- tools=event.tools,- headers=event.headers,- response=event.response,- metadata={**event.metadata, "replaced": True},- )-- class BrokenExtension:- def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201- raise RuntimeError("boom")-- hook = Hook()- monkeypatch.setattr(- "headroom.pipeline.discover_pipeline_extensions",- lambda: [BrokenExtension()],- )-- manager = PipelineExtensionManager(- hooks=hook,- extensions=[object(), ReplacingExtension()],- discover=True,- )-- assert manager.enabled is True- event = manager.emit(- PipelineStage.INPUT_RECEIVED,- operation="compress",- request_id="req-1",- provider="openai",- model="gpt-4o",- messages=[{"role": "user", "content": "hello"}],- metadata={"start": True},- )-- assert hook.seen == ["input_received"]- assert event.metadata == {"start": True, "hook": True, "replaced": True}- assert event.request_id == "req-1"-- disabled = PipelineExtensionManager(discover=False)- assert disabled.enabled is False-- assert summarize_routing_markers(["router:smart", "other", "router:cheap"]) == [- "router:smart",- "router:cheap",- ]- assert PipelineStage.SETUP in CANONICAL_PIPELINE_STAGES- assert PipelineStage.RESPONSE_RECEIVED in CANONICAL_PIPELINE_STAGES+from __future__ import annotations++import importlib.metadata+from dataclasses import dataclass++from headroom.pipeline import (+ CANONICAL_PIPELINE_STAGES,+ ENTRY_POINT_GROUP,+ PipelineEvent,+ PipelineExtensionManager,+ PipelineStage,+ discover_pipeline_extensions,+ summarize_routing_markers,+)+++@dataclass+class FakeEntryPoint:+ name: str+ value: object++ def load(self):+ if isinstance(self.value, Exception):+ raise self.value+ return self.value+++def test_discover_pipeline_extensions_handles_load_and_init_failures(+ monkeypatch,+) -> None:+ class WorkingExtension:+ def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201+ return event++ class NeedsInit:+ def __init__(self) -> None:+ raise RuntimeError("bad init")++ monkeypatch.setattr(+ importlib.metadata,+ "entry_points",+ lambda group=None: [+ FakeEntryPoint("working-instance", WorkingExtension()),+ FakeEntryPoint("working-class", WorkingExtension),+ FakeEntryPoint("bad-load", RuntimeError("bad load")),+ FakeEntryPoint("bad-init", NeedsInit),+ ]+ if group == ENTRY_POINT_GROUP+ else [],+ )++ discovered = discover_pipeline_extensions()+ assert len(discovered) == 2+ assert all(callable(getattr(ext, "on_pipeline_event", None)) for ext in discovered)+++def test_discover_pipeline_extensions_handles_enumeration_failure(monkeypatch) -> None:+ monkeypatch.setattr(+ importlib.metadata,+ "entry_points",+ lambda group=None: (_ for _ in ()).throw(RuntimeError("boom")),+ )+ assert discover_pipeline_extensions() == []+++def test_pipeline_manager_emit_and_summary(monkeypatch) -> None:+ class Hook:+ def __init__(self) -> None:+ self.seen: list[str] = []++ def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201+ self.seen.append(event.stage.value)+ event.metadata["hook"] = True+ return event++ class ReplacingExtension:+ def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201+ return PipelineEvent(+ stage=event.stage,+ operation=event.operation,+ request_id=event.request_id,+ provider=event.provider,+ model=event.model,+ messages=event.messages,+ tools=event.tools,+ headers=event.headers,+ response=event.response,+ metadata={**event.metadata, "replaced": True},+ )++ class BrokenExtension:+ def on_pipeline_event(self, event: PipelineEvent): # noqa: ANN001, ANN201+ raise RuntimeError("boom")++ hook = Hook()+ monkeypatch.setattr(+ "headroom.pipeline.discover_pipeline_extensions",+ lambda: [BrokenExtension()],+ )++ manager = PipelineExtensionManager(+ hooks=hook,+ extensions=[object(), ReplacingExtension()],+ discover=True,+ )++ assert manager.enabled is True+ event = manager.emit(+ PipelineStage.INPUT_RECEIVED,+ operation="compress",+ request_id="req-1",+ provider="openai",+ model="gpt-4o",+ messages=[{"role": "user", "content": "hello"}],+ metadata={"start": True},+ )++ assert hook.seen == ["input_received"]+ assert event.metadata == {"start": True, "hook": True, "replaced": True}+ assert event.request_id == "req-1"++ disabled = PipelineExtensionManager(discover=False)+ assert disabled.enabled is False++ assert summarize_routing_markers(["router:smart", "other", "router:cheap"]) == [+ "router:smart",+ "router:cheap",+ ]+ assert PipelineStage.SETUP in CANONICAL_PIPELINE_STAGES+ assert PipelineStage.RESPONSE_RECEIVED in CANONICAL_PIPELINE_STAGESdiff --git a/tests/test_proxy_handlers_batch.py b/tests/test_proxy_handlers_batch.pyindex f082d2755..0fa2453cc 100644--- a/tests/test_proxy_handlers_batch.py+++ b/tests/test_proxy_handlers_batch.py@@ -1,1047 +1,1047 @@-from __future__ import annotations--import json-import sys-from types import SimpleNamespace--import pytest--from headroom.proxy.handlers import batch as batch_module---class FakeResponse:- def __init__(- self,- *,- status_code: int = 200,- content: bytes = b"{}",- headers: dict[str, str] | None = None,- text: str | None = None,- json_data=None, # noqa: ANN001- ) -> None:- self.status_code = status_code- self.content = content- self.headers = headers or {}- self.text = text if text is not None else content.decode("utf-8", errors="ignore")- self._json_data = json_data-- def json(self): # noqa: ANN201- if self._json_data is not None:- return self._json_data- return json.loads(self.text)---class FakeHttpClient:- def __init__(self) -> None:- self.posts: list[dict[str, object]] = []- self.gets: list[dict[str, object]] = []- self.requests: list[dict[str, object]] = []- self.post_response = FakeResponse()- self.get_response = FakeResponse()- self.raise_post: Exception | None = None- self.raise_get: Exception | None = None-- async def post(self, url: str, **kwargs): # noqa: ANN003, ANN201- self.posts.append({"url": url, **kwargs})- if self.raise_post is not None:- raise self.raise_post- return self.post_response-- async def get(self, url: str, **kwargs): # noqa: ANN003, ANN201- self.gets.append({"url": url, **kwargs})- if self.raise_get is not None:- raise self.raise_get- return self.get_response-- async def request(self, method: str, url: str, **kwargs): # noqa: ANN003, ANN201- self.requests.append({"method": method, "url": url, **kwargs})- if self.raise_get is not None:- raise self.raise_get- return self.get_response---class FakeMetrics:- def __init__(self) -> None:- self.record_calls: list[dict[str, object]] = []- self.failed_calls: list[dict[str, object]] = []-- async def record_request(self, **kwargs) -> None: # noqa: ANN003- self.record_calls.append(kwargs)-- async def record_failed(self, **kwargs) -> None: # noqa: ANN003- self.failed_calls.append(kwargs)---class DummyBatchHandler(batch_module.BatchHandlerMixin):- OPENAI_API_URL = "https://openai.example"- GEMINI_API_URL = "https://gemini.example"-- def __init__(self) -> None:- self.http_client = FakeHttpClient()- self.metrics = FakeMetrics()- self.config = SimpleNamespace(- optimize=False,- ccr_inject_tool=False,- ccr_inject_system_instructions=False,- )- self.openai_provider = SimpleNamespace(get_context_limit=lambda model: 8192)- self.openai_pipeline = SimpleNamespace(apply=lambda **kwargs: None)- self._request_counter = 0- self._retry_response = FakeResponse()-- async def _next_request_id(self) -> str:- self._request_counter += 1- return f"req-{self._request_counter}"-- async def handle_passthrough(self, request, base_url): # noqa: ANN001, ANN201- return {"request": request, "base_url": base_url}-- async def _retry_request(self, method, url, headers, body): # noqa: ANN001, ANN201- return self._retry_response-- def _gemini_contents_to_messages(self, contents, system_instruction): # noqa: ANN001, ANN201- messages = [{"role": "user", "content": part["parts"][0]["text"]} for part in contents]- return messages, []-- def _messages_to_gemini_contents(self, messages): # noqa: ANN001, ANN201- return ([{"parts": [{"text": message["content"]}]} for message in messages], None)---class FakeRequest:- def __init__(- self,- body: bytes | str,- *,- headers: dict[str, str] | None = None,- method: str = "POST",- path: str = "/v1/batches",- query: str = "",- ) -> None:- self._body = body.encode("utf-8") if isinstance(body, str) else body- self.headers = headers or {}- self.method = method- self.url = SimpleNamespace(path=path, query=query)-- async def body(self) -> bytes:- return self._body---def install_batch_support_modules(- monkeypatch: pytest.MonkeyPatch,- *,- injector_result=None, # noqa: ANN001- tokenizer_count: int = 10,-) -> None:- class FakeInjector:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs-- def process_request(self, messages, tools): # noqa: ANN001, ANN201- if injector_result is not None:- return injector_result- return messages, tools, False-- class FakeTokenizer:- def count_messages(self, messages) -> int: # noqa: ANN001- return tokenizer_count-- monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))- monkeypatch.setitem(- sys.modules,- "headroom.tokenizers",- SimpleNamespace(get_tokenizer=lambda model: FakeTokenizer()),- )- monkeypatch.setitem(- sys.modules,- "headroom.utils",- SimpleNamespace(extract_user_query=lambda messages: "query"),- )---async def test_compress_batch_jsonl_without_optimization_handles_invalid_lines(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(monkeypatch, tokenizer_count=12)- handler = DummyBatchHandler()- content = "\n".join(- [- json.dumps(- {"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}}- ),- json.dumps({"body": {"model": "gpt-4o", "messages": []}}),- "not-json",- ]- )-- lines, stats = await handler._compress_batch_jsonl(content, "req-1")-- assert len(lines) == 3- assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hi"- assert lines[2] == "not-json"- assert stats == {- "total_requests": 3,- "total_original_tokens": 12,- "total_compressed_tokens": 12,- "total_tokens_saved": 0,- "savings_percent": 0.0,- "errors": 1,- }---async def test_compress_batch_jsonl_uses_pipeline_and_ccr_injection(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(- monkeypatch,- injector_result=(- [{"role": "system", "content": "compressed"}],- [{"name": "retrieval"}],- True,- ),- )- handler = DummyBatchHandler()- handler.config.optimize = True- handler.config.ccr_inject_tool = True- handler.openai_pipeline = SimpleNamespace(- apply=lambda **kwargs: SimpleNamespace(- messages=[{"role": "assistant", "content": "short"}],- tokens_before=100,- tokens_after=40,- )- )-- lines, stats = await handler._compress_batch_jsonl(- json.dumps(- {- "body": {- "model": "gpt-4o-mini",- "messages": [{"role": "user", "content": "hello"}],- "tools": [{"name": "existing"}],- }- }- ),- "req-2",- )-- body = json.loads(lines[0])["body"]- assert body["messages"] == [{"role": "system", "content": "compressed"}]- assert body["tools"] == [{"name": "retrieval"}]- assert stats["total_tokens_saved"] == 60- assert stats["savings_percent"] == 60.0---async def test_compress_batch_jsonl_falls_back_when_pipeline_raises(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(monkeypatch, tokenizer_count=33)- handler = DummyBatchHandler()- handler.config.optimize = True- handler.openai_pipeline = SimpleNamespace(- apply=lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))- )-- lines, stats = await handler._compress_batch_jsonl(- json.dumps({"body": {"messages": [{"role": "user", "content": "hello"}]}}),- "req-3",- )-- assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hello"- assert stats["total_original_tokens"] == 33- assert stats["total_compressed_tokens"] == 33---async def test_batch_passthrough_forwards_request_and_strips_response_headers() -> None:- handler = DummyBatchHandler()- handler.http_client.post_response = FakeResponse(- content=b'{"ok":true}',- headers={"content-encoding": "gzip", "content-length": "20", "x-kept": "1"},- )-- response = await handler._batch_passthrough(- FakeRequest(- '{"input_file_id":"file-1"}', headers={"host": "example", "content-length": "10"}- ),- {"input_file_id": "file-1"},- )-- assert response.status_code == 200- assert dict(response.headers)["x-kept"] == "1"- assert "content-encoding" not in dict(response.headers)- assert handler.http_client.posts[0]["url"] == "https://openai.example/v1/batches"---async def test_handle_batch_create_validates_json_and_required_fields(- monkeypatch: pytest.MonkeyPatch,-) -> None:- handler = DummyBatchHandler()-- async def raise_bad_json(request): # noqa: ANN001- raise ValueError("bad json")-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", raise_bad_json)-- bad = await handler.handle_batch_create(FakeRequest("{}"))- assert bad.status_code == 400- assert bad.body.decode().find("invalid_json") > 0-- async def missing_file_payload(request): # noqa: ANN001- return {"endpoint": "/v1/chat/completions"}-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_file_payload)- missing_file = await handler.handle_batch_create(FakeRequest("{}"))- assert missing_file.status_code == 400- assert missing_file.body.decode().find("input_file_id is required") > 0-- async def missing_endpoint_payload(request): # noqa: ANN001- return {"input_file_id": "file-1"}-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_endpoint_payload)- missing_endpoint = await handler.handle_batch_create(FakeRequest("{}"))- assert missing_endpoint.status_code == 400- assert missing_endpoint.body.decode().find("endpoint is required") > 0---async def test_handle_batch_create_passthrough_and_download_failure(- monkeypatch: pytest.MonkeyPatch,-) -> None:- handler = DummyBatchHandler()- passthrough_response = SimpleNamespace(marker="passthrough")-- async def fake_passthrough(request, body): # noqa: ANN001- return passthrough_response-- monkeypatch.setattr(handler, "_batch_passthrough", fake_passthrough)-- async def passthrough_payload(request): # noqa: ANN001- return {"input_file_id": "file-1", "endpoint": "/v1/responses"}-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", passthrough_payload)- assert await handler.handle_batch_create(FakeRequest("{}")) is passthrough_response-- async def download_missing_payload(request): # noqa: ANN001- return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}-- async def missing_download(file_id, headers): # noqa: ANN001- return None-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", download_missing_payload)- monkeypatch.setattr(handler, "_download_openai_file", missing_download)- missing = await handler.handle_batch_create(FakeRequest("{}"))- assert missing.status_code == 404- assert missing.body.decode().find("file_not_found") > 0---async def test_handle_batch_create_handles_empty_upload_failure_and_success(- monkeypatch: pytest.MonkeyPatch,-) -> None:- handler = DummyBatchHandler()-- async def request_payload(request): # noqa: ANN001- return {- "input_file_id": "file-1",- "endpoint": "/v1/chat/completions",- "completion_window": "12h",- "metadata": {"source": "test"},- }-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)-- async def fake_download(file_id, headers): # noqa: ANN001- return "downloaded"-- monkeypatch.setattr(handler, "_download_openai_file", fake_download)-- async def empty_compress(content, request_id): # noqa: ANN001- return [], {- "total_requests": 0,- "total_original_tokens": 0,- "total_compressed_tokens": 0,- "total_tokens_saved": 0,- "savings_percent": 0.0,- "errors": 0,- }-- monkeypatch.setattr(handler, "_compress_batch_jsonl", empty_compress)- empty = await handler.handle_batch_create(FakeRequest("{}"))- assert empty.status_code == 400- assert empty.body.decode().find("empty_file") > 0-- async def compressed(content, request_id): # noqa: ANN001- return ['{"body":{}}'], {- "total_requests": 1,- "total_original_tokens": 20,- "total_compressed_tokens": 10,- "total_tokens_saved": 10,- "savings_percent": 50.0,- "errors": 0,- }-- monkeypatch.setattr(handler, "_compress_batch_jsonl", compressed)-- async def upload_failed_file(content, filename, headers): # noqa: ANN001- return None-- monkeypatch.setattr(handler, "_upload_openai_file", upload_failed_file)- upload_failed = await handler.handle_batch_create(FakeRequest("{}"))- assert upload_failed.status_code == 500- assert upload_failed.body.decode().find("upload_failed") > 0-- handler.http_client.post_response = FakeResponse(- content=b'{"id":"batch_123","object":"batch"}',- headers={"content-encoding": "gzip", "content-length": "12", "x-openai": "1"},- )-- async def upload_success(content, filename, headers): # noqa: ANN001- return "file-compressed"-- monkeypatch.setattr(handler, "_upload_openai_file", upload_success)- success = await handler.handle_batch_create(- FakeRequest(- "{}", headers={"host": "proxy", "content-length": "4", "authorization": "Bearer test"}- )- )-- assert success.status_code == 200- success_headers = dict(success.headers)- assert success_headers["x-headroom-tokens-saved"] == "10"- assert success_headers["x-headroom-savings-percent"] == "50.0"- assert success_headers["x-openai"] == "1"- sent_body = handler.http_client.posts[-1]["json"]- assert sent_body["metadata"]["headroom_compressed"] == "true"- assert sent_body["metadata"]["headroom_original_file_id"] == "file-1"- assert handler.metrics.record_calls[-1]["provider"] == "openai"---async def test_handle_batch_create_records_failure_on_exception(- monkeypatch: pytest.MonkeyPatch,-) -> None:- handler = DummyBatchHandler()-- async def request_payload(request): # noqa: ANN001- return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}-- async def boom(file_id, headers): # noqa: ANN001- raise RuntimeError("boom")-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)- monkeypatch.setattr(handler, "_download_openai_file", boom)-- response = await handler.handle_batch_create(FakeRequest("{}"))-- assert response.status_code == 500- assert handler.metrics.failed_calls == [{"provider": "batch"}]---async def test_download_and_upload_openai_file_helpers() -> None:- handler = DummyBatchHandler()- handler.http_client.get_response = FakeResponse(status_code=200, text="jsonl-content")- downloaded = await handler._download_openai_file("file-1", {"authorization": "Bearer token"})- assert downloaded == "jsonl-content"- assert handler.http_client.gets[0]["url"] == "https://openai.example/v1/files/file-1/content"-- handler.http_client.get_response = FakeResponse(status_code=404, text="missing")- assert await handler._download_openai_file("file-2", {}) is None-- handler.http_client.post_response = FakeResponse(- status_code=200,- json_data={"id": "file-uploaded"},- headers={"content-type": "application/json"},- )- file_id = await handler._upload_openai_file(- '{"body":{}}',- "compressed.jsonl",- {"authorization": "Bearer token", "content-type": "application/json"},- )- assert file_id == "file-uploaded"- post_call = handler.http_client.posts[-1]- assert post_call["headers"] == {"authorization": "Bearer token"}- assert post_call["files"]["file"][0] == "compressed.jsonl"-- handler.http_client.post_response = FakeResponse(status_code=500, text="fail")- assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None- handler.http_client.raise_post = RuntimeError("network")- assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None---async def test_store_google_batch_context_persists_transformed_requests(- monkeypatch: pytest.MonkeyPatch,-) -> None:- stored_contexts: list[object] = []-- class FakeBatchContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs- self.requests: list[object] = []-- def add_request(self, request) -> None: # noqa: ANN001- self.requests.append(request)-- class FakeBatchRequestContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs-- class FakeStore:- async def store(self, context) -> None: # noqa: ANN001- stored_contexts.append(context)-- monkeypatch.setitem(- sys.modules,- "headroom.ccr",- SimpleNamespace(- BatchContext=FakeBatchContext,- BatchRequestContext=FakeBatchRequestContext,- get_batch_context_store=lambda: FakeStore(),- ),- )-- handler = DummyBatchHandler()- await handler._store_google_batch_context(- "batches/123",- [- {- "metadata": {"key": "req-1"},- "request": {- "contents": [{"parts": [{"text": "hello"}]}],- "systemInstruction": {"parts": [{"text": "system"}]},- "tools": [{"name": "tool"}],- },- }- ],- "gemini-2.0",- "api-key",- )-- context = stored_contexts[0]- assert context.kwargs["batch_id"] == "batches/123"- assert context.requests[0].kwargs["custom_id"] == "req-1"- assert context.requests[0].kwargs["messages"] == [{"role": "user", "content": "hello"}]- assert context.requests[0].kwargs["system_instruction"] == "system"---async def test_handle_google_batch_results_passes_through_early_exit_cases(- monkeypatch: pytest.MonkeyPatch,-) -> None:- class FakeStore:- async def get(self, batch_name): # noqa: ANN001- return None-- monkeypatch.setitem(- sys.modules,- "headroom.ccr",- SimpleNamespace(- BatchResultProcessor=lambda http_client: None,- get_batch_context_store=lambda: FakeStore(),- ),- )-- handler = DummyBatchHandler()- request = FakeRequest(- "{}", headers={"x-goog-api-key": "secret"}, method="GET", path="/v1beta/batches/b1"- )-- handler.http_client.get_response = FakeResponse(- status_code=500, content=b"bad", headers={"x-upstream": "1"}- )- error_response = await handler.handle_google_batch_results(request, "batches/b1")- assert error_response.status_code == 500- assert dict(error_response.headers)["x-upstream"] == "1"-- class BadJsonResponse(FakeResponse):- def json(self): # noqa: ANN201- raise json.JSONDecodeError("bad", "x", 0)-- handler.http_client.get_response = BadJsonResponse(- status_code=200, content=b"plain", headers={"x-upstream": "2"}- )- non_json = await handler.handle_google_batch_results(request, "batches/b1")- assert non_json.status_code == 200- assert dict(non_json.headers)["x-upstream"] == "2"-- handler.http_client.get_response = FakeResponse(- status_code=200,- content=b"{}",- json_data={"metadata": {"state": "RUNNING"}},- )- running = await handler.handle_google_batch_results(request, "batches/b1")- assert running.status_code == 200-- handler.http_client.get_response = FakeResponse(- status_code=200,- content=b"{}",- json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": []}},- )- no_results = await handler.handle_google_batch_results(request, "batches/b1")- assert no_results.status_code == 200-- handler.http_client.get_response = FakeResponse(- status_code=200,- content=b"{}",- json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": [{"id": 1}]}},- )- handler.config.ccr_inject_tool = False- no_ccr = await handler.handle_google_batch_results(request, "batches/b1")- assert no_ccr.status_code == 200- assert "key=secret" in handler.http_client.gets[-1]["url"]---async def test_handle_google_batch_results_processes_completed_results(- monkeypatch: pytest.MonkeyPatch,-) -> None:- processed_calls: list[tuple[str, list[object], str]] = []-- class FakeProcessed:- def __init__(- self, result, custom_id: str, was_processed: bool, continuation_rounds: int- ) -> None: # noqa: ANN001- self.result = result- self.custom_id = custom_id- self.was_processed = was_processed- self.continuation_rounds = continuation_rounds-- class FakeProcessor:- def __init__(self, http_client) -> None: # noqa: ANN001- self.http_client = http_client-- async def process_results(self, batch_name, results, provider): # noqa: ANN001- processed_calls.append((batch_name, results, provider))- return [- FakeProcessed({"id": "processed"}, "req-1", True, 2),- FakeProcessed({"id": "unchanged"}, "req-2", False, 0),- ]-- class FakeStore:- async def get(self, batch_name): # noqa: ANN001- return SimpleNamespace(batch_name=batch_name)-- monkeypatch.setitem(- sys.modules,- "headroom.ccr",- SimpleNamespace(- BatchResultProcessor=FakeProcessor,- get_batch_context_store=lambda: FakeStore(),- ),- )-- handler = DummyBatchHandler()- handler.config.ccr_inject_tool = True- handler.http_client.get_response = FakeResponse(- status_code=200,- content=b"{}",- json_data={- "metadata": {"state": "SUCCEEDED"},- "response": {"responses": [{"id": "raw-1"}, {"id": "raw-2"}]},- },- )-- response = await handler.handle_google_batch_results(- FakeRequest("{}", method="GET", path="/v1beta/batches/b1"),- "batches/b1",- )-- payload = json.loads(response.body)- assert payload["response"]["responses"] == [{"id": "processed"}, {"id": "unchanged"}]- assert processed_calls == [("batches/b1", [{"id": "raw-1"}, {"id": "raw-2"}], "google")]- assert handler.metrics.record_calls[-1]["model"] == "batch:ccr-processed"---async def test_google_batch_passthrough_helpers_forward_and_track_metrics() -> None:- handler = DummyBatchHandler()- handler.http_client.post_response = FakeResponse(- content=b'{"ok":true}',- headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},- )- handler.http_client.post_response = FakeResponse(- content=b'{"ok":true}',- headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},- )-- passthrough = await handler._google_batch_passthrough(- FakeRequest(- "body", headers={"host": "proxy", "content-length": "4", "x-goog-api-key": "secret"}- ),- "gemini-pro",- {"batch": {}},- )- assert passthrough.status_code == 200- assert dict(passthrough.headers)["x-kept"] == "1"- assert "key=secret" in handler.http_client.posts[-1]["url"]- assert handler.metrics.record_calls[-1]["model"] == "passthrough:batch:gemini-pro"-- handler.http_client.get_response = FakeResponse(- content=b'{"state":"ok"}',- headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "2"},- )- response = await handler.handle_google_batch_passthrough(- FakeRequest(- "ping",- headers={"host": "proxy", "x-goog-api-key": "secret"},- method="DELETE",- path="/v1beta/batches/b1",- query="alt=json",- ),- "b1",- )- assert response.status_code == 200- assert dict(response.headers)["x-kept"] == "2"- get_call = handler.http_client.requests[-1]- assert get_call["url"] == "https://gemini.example/v1beta/batches/b1?alt=json&key=secret"- assert handler.metrics.record_calls[-1]["model"] == "passthrough:batches"---async def test_handle_google_batch_create_validates_and_passthroughs(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(monkeypatch)- handler = DummyBatchHandler()-- too_large = await handler.handle_google_batch_create(- FakeRequest("{}", headers={"content-length": str(200 * 1024 * 1024)}),- "gemini-pro",- )- assert too_large.status_code == 413-- async def bad_json(request): # noqa: ANN001- raise ValueError("bad json")-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", bad_json)- invalid = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")- assert invalid.status_code == 400-- passthrough_response = SimpleNamespace(kind="passthrough")-- async def fake_google_passthrough(request, model, body=None): # noqa: ANN001- return passthrough_response-- async def no_inline(request): # noqa: ANN001- return {"batch": {"input_config": {"requests": {"requests": []}}}}-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", no_inline)- monkeypatch.setattr(handler, "_google_batch_passthrough", fake_google_passthrough)- assert (- await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")- is passthrough_response- )---async def test_handle_google_batch_create_success_and_failure_paths(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(monkeypatch)- handler = DummyBatchHandler()- handler.config.optimize = True- handler.config.ccr_inject_tool = True- handler.openai_pipeline = SimpleNamespace(- apply=lambda **kwargs: SimpleNamespace(- messages=[{"role": "user", "content": "compressed"}],- timing={"compress": 1.2},- tokens_before=100,- tokens_after=40,- )- )-- class FakeInjector:- def __init__(self, **kwargs) -> None: # noqa: ANN003- pass-- def process_request(self, messages, tools): # noqa: ANN001, ANN201- return (- messages + [{"role": "system", "content": "retrieval"}],- [{"name": "retrieval"}],- True,- )-- monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))-- stored: list[tuple[str, list[dict[str, object]], str, str | None]] = []-- async def fake_store(batch_name, requests_list, model, api_key): # noqa: ANN001- stored.append((batch_name, requests_list, model, api_key))-- async def fake_retry(method, url, headers, body): # noqa: ANN001- return FakeResponse(- status_code=200,- content=b'{"name":"batches/123"}',- headers={"content-encoding": "gzip", "content-length": "10", "x-upstream": "1"},- json_data={"name": "batches/123"},- )-- async def good_payload(request): # noqa: ANN001- return {- "batch": {- "input_config": {- "requests": {- "requests": [- {- "request": {- "contents": [{"parts": [{"text": "hello"}]}],- "tools": [{"functionDeclarations": [{"name": "existing"}]}],- },- "metadata": {"key": "req-1"},- }- ]- }- }- }- }-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", good_payload)- monkeypatch.setattr(handler, "_retry_request", fake_retry)- monkeypatch.setattr(handler, "_store_google_batch_context", fake_store)-- response = await handler.handle_google_batch_create(- FakeRequest("{}", headers={"x-goog-api-key": "secret"}),- "gemini-pro",- )- assert response.status_code == 200- assert dict(response.headers)["x-upstream"] == "1"- assert handler.metrics.record_calls[-1]["provider"] == "google"- assert handler.metrics.record_calls[-1]["tokens_saved"] == 60- assert stored[0][0] == "batches/123"- assert stored[0][2:] == ("gemini-pro", "secret")- assert stored[0][1][0]["metadata"] == {"key": "req-1"}-- async def broken_retry(method, url, headers, body): # noqa: ANN001- raise RuntimeError("forward failed")-- monkeypatch.setattr(handler, "_retry_request", broken_retry)- failed = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")- assert failed.status_code == 500---async def test_handle_google_batch_create_covers_passthrough_revert_and_store_failures(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(- monkeypatch, injector_result=([{"role": "user", "content": "kept"}], None, False)- )- handler = DummyBatchHandler()- handler.config.optimize = True- handler.config.ccr_inject_tool = True-- pipeline_calls: list[dict[str, object]] = []- handler.openai_pipeline = SimpleNamespace(- apply=lambda **kwargs: pipeline_calls.append(kwargs)- or SimpleNamespace(- messages=[{"role": "user", "content": "inflated"}],- timing={},- tokens_before=40,- tokens_after=80,- )- )-- def fake_to_messages(contents, system_instruction): # noqa: ANN001, ANN201- if contents and "inlineData" in contents[0]["parts"][0]:- return ([{"role": "user", "content": "binary"}], [0])- return ([{"role": "user", "content": "compress"}], [])-- def fake_to_gemini(messages): # noqa: ANN001, ANN201- return ([{"parts": [{"text": "new"}]}], {"parts": [{"text": "sys"}]})-- async def payload(request): # noqa: ANN001- return {- "batch": {- "input_config": {- "requests": {- "requests": [- {"request": {"contents": []}, "metadata": {"key": "empty"}},- {- "request": {"contents": [{"parts": [{"inlineData": "x"}]}]},- "metadata": {"key": "preserved"},- },- {- "request": {- "contents": [{"parts": [{"text": "hello"}]}],- "tools": [- {"other": True},- {"functionDeclarations": [{"name": "existing"}]},- ],- },- "metadata": {"key": "optimized"},- },- ]- }- }- }- }-- seen_bodies: list[dict[str, object]] = []-- async def retry(method, url, headers, body): # noqa: ANN001- seen_bodies.append(body)- return FakeResponse(status_code=200, content=b"{}", json_data={"name": "batches/123"})-- async def broken_store(batch_name, requests_list, model, api_key): # noqa: ANN001- raise RuntimeError("store failed")-- monkeypatch.setattr("headroom.proxy.helpers._read_request_json", payload)- monkeypatch.setattr(handler, "_gemini_contents_to_messages", fake_to_messages)- monkeypatch.setattr(handler, "_messages_to_gemini_contents", fake_to_gemini)- monkeypatch.setattr(handler, "_retry_request", retry)- monkeypatch.setattr(handler, "_store_google_batch_context", broken_store)-- response = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")- assert response.status_code == 200- assert len(pipeline_calls) == 1- assert handler.metrics.record_calls[-1]["tokens_saved"] == 0- assert (- seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][0]["metadata"]["key"]- == "empty"- )- optimized = seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][2]["request"]- assert optimized["contents"][0] == {"parts": [{"text": "new"}]}- assert optimized["systemInstruction"] == {"parts": [{"text": "sys"}]}---async def test_google_batch_passthrough_without_body_and_query_variants() -> None:- handler = DummyBatchHandler()- handler.http_client.post_response = FakeResponse(content=b"ok", headers={"x-upstream": "1"})-- response = await handler._google_batch_passthrough(- FakeRequest("raw-body", headers={"host": "proxy"}, method="POST"),- "gemini-pro",- )- assert response.status_code == 200- assert handler.http_client.posts[-1]["content"] == b"raw-body"-- handler.http_client.get_response = FakeResponse(content=b"{}", headers={"x-upstream": "2"})- passthrough = await handler.handle_google_batch_passthrough(- FakeRequest(- "{}",- headers={"host": "proxy", "x-goog-api-key": "secret"},- method="GET",- path="/v1beta/batches/b1",- ),- "b1",- )- assert passthrough.status_code == 200- assert (- handler.http_client.requests[-1]["url"]- == "https://gemini.example/v1beta/batches/b1?key=secret"- )---async def test_batch_helper_methods_and_openai_file_error_branches() -> None:- handler = DummyBatchHandler()- marker = object()-- async def fake_passthrough(request, base_url): # noqa: ANN001- return marker-- handler.handle_passthrough = fake_passthrough- request = FakeRequest("{}")- assert await handler.handle_batch_list(request) is marker- assert await handler.handle_batch_get(request, "b1") is marker- assert await handler.handle_batch_cancel(request, "b1") is marker-- handler.http_client.raise_get = RuntimeError("download boom")- assert await handler._download_openai_file("file-1", {}) is None-- handler.http_client.raise_get = None- handler.http_client.post_response = FakeResponse(status_code=200, json_data={})- assert await handler._upload_openai_file("{}", "missing-id.jsonl", {}) is None---async def test_store_google_batch_context_without_system_text() -> None:- stored_contexts: list[object] = []-- class FakeBatchContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs- self.requests: list[object] = []-- def add_request(self, request) -> None: # noqa: ANN001- self.requests.append(request)-- class FakeBatchRequestContext:- def __init__(self, **kwargs) -> None: # noqa: ANN003- self.kwargs = kwargs-- class FakeStore:- async def store(self, context) -> None: # noqa: ANN001- stored_contexts.append(context)-- handler = DummyBatchHandler()- sys.modules["headroom.ccr"] = SimpleNamespace(- BatchContext=FakeBatchContext,- BatchRequestContext=FakeBatchRequestContext,- get_batch_context_store=lambda: FakeStore(),- )-- await handler._store_google_batch_context(- "batches/456",- [- {- "request": {- "contents": [{"parts": [{"text": "hello"}]}],- "systemInstruction": {"parts": ["bad"]},- }- }- ],- "gemini-2.0",- None,- )-- context = stored_contexts[0]- assert context.kwargs["api_key"] is None- assert context.requests[0].kwargs["custom_id"] == ""- assert context.requests[0].kwargs["system_instruction"] is None---async def test_compress_batch_jsonl_skips_blank_lines_and_preserves_tools_when_not_injected(- monkeypatch: pytest.MonkeyPatch,-) -> None:- install_batch_support_modules(- monkeypatch,- injector_result=([{"role": "assistant", "content": "short"}], [{"name": "orig"}], False),- )- handler = DummyBatchHandler()- handler.config.optimize = True- handler.config.ccr_inject_tool = True- handler.openai_pipeline = SimpleNamespace(- apply=lambda **kwargs: SimpleNamespace(- messages=[{"role": "assistant", "content": "short"}],- tokens_before=50,- tokens_after=10,- )- )-- lines, stats = await handler._compress_batch_jsonl(- "\n"- + json.dumps(- {- "body": {- "model": "gpt-4o",- "messages": [{"role": "user", "content": "hello"}],- "tools": [{"name": "orig"}],- }- }- )- + "\n",- "req-extra",- )-- assert len(lines) == 1- body = json.loads(lines[0])["body"]- assert body["tools"] == [{"name": "orig"}]- assert stats["total_requests"] == 1- assert stats["errors"] == 0+from __future__ import annotations++import json+import sys+from types import SimpleNamespace++import pytest++from headroom.proxy.handlers import batch as batch_module+++class FakeResponse:+ def __init__(+ self,+ *,+ status_code: int = 200,+ content: bytes = b"{}",+ headers: dict[str, str] | None = None,+ text: str | None = None,+ json_data=None, # noqa: ANN001+ ) -> None:+ self.status_code = status_code+ self.content = content+ self.headers = headers or {}+ self.text = text if text is not None else content.decode("utf-8", errors="ignore")+ self._json_data = json_data++ def json(self): # noqa: ANN201+ if self._json_data is not None:+ return self._json_data+ return json.loads(self.text)+++class FakeHttpClient:+ def __init__(self) -> None:+ self.posts: list[dict[str, object]] = []+ self.gets: list[dict[str, object]] = []+ self.requests: list[dict[str, object]] = []+ self.post_response = FakeResponse()+ self.get_response = FakeResponse()+ self.raise_post: Exception | None = None+ self.raise_get: Exception | None = None++ async def post(self, url: str, **kwargs): # noqa: ANN003, ANN201+ self.posts.append({"url": url, **kwargs})+ if self.raise_post is not None:+ raise self.raise_post+ return self.post_response++ async def get(self, url: str, **kwargs): # noqa: ANN003, ANN201+ self.gets.append({"url": url, **kwargs})+ if self.raise_get is not None:+ raise self.raise_get+ return self.get_response++ async def request(self, method: str, url: str, **kwargs): # noqa: ANN003, ANN201+ self.requests.append({"method": method, "url": url, **kwargs})+ if self.raise_get is not None:+ raise self.raise_get+ return self.get_response+++class FakeMetrics:+ def __init__(self) -> None:+ self.record_calls: list[dict[str, object]] = []+ self.failed_calls: list[dict[str, object]] = []++ async def record_request(self, **kwargs) -> None: # noqa: ANN003+ self.record_calls.append(kwargs)++ async def record_failed(self, **kwargs) -> None: # noqa: ANN003+ self.failed_calls.append(kwargs)+++class DummyBatchHandler(batch_module.BatchHandlerMixin):+ OPENAI_API_URL = "https://openai.example"+ GEMINI_API_URL = "https://gemini.example"++ def __init__(self) -> None:+ self.http_client = FakeHttpClient()+ self.metrics = FakeMetrics()+ self.config = SimpleNamespace(+ optimize=False,+ ccr_inject_tool=False,+ ccr_inject_system_instructions=False,+ )+ self.openai_provider = SimpleNamespace(get_context_limit=lambda model: 8192)+ self.openai_pipeline = SimpleNamespace(apply=lambda **kwargs: None)+ self._request_counter = 0+ self._retry_response = FakeResponse()++ async def _next_request_id(self) -> str:+ self._request_counter += 1+ return f"req-{self._request_counter}"++ async def handle_passthrough(self, request, base_url): # noqa: ANN001, ANN201+ return {"request": request, "base_url": base_url}++ async def _retry_request(self, method, url, headers, body): # noqa: ANN001, ANN201+ return self._retry_response++ def _gemini_contents_to_messages(self, contents, system_instruction): # noqa: ANN001, ANN201+ messages = [{"role": "user", "content": part["parts"][0]["text"]} for part in contents]+ return messages, []++ def _messages_to_gemini_contents(self, messages): # noqa: ANN001, ANN201+ return ([{"parts": [{"text": message["content"]}]} for message in messages], None)+++class FakeRequest:+ def __init__(+ self,+ body: bytes | str,+ *,+ headers: dict[str, str] | None = None,+ method: str = "POST",+ path: str = "/v1/batches",+ query: str = "",+ ) -> None:+ self._body = body.encode("utf-8") if isinstance(body, str) else body+ self.headers = headers or {}+ self.method = method+ self.url = SimpleNamespace(path=path, query=query)++ async def body(self) -> bytes:+ return self._body+++def install_batch_support_modules(+ monkeypatch: pytest.MonkeyPatch,+ *,+ injector_result=None, # noqa: ANN001+ tokenizer_count: int = 10,+) -> None:+ class FakeInjector:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ self.kwargs = kwargs++ def process_request(self, messages, tools): # noqa: ANN001, ANN201+ if injector_result is not None:+ return injector_result+ return messages, tools, False++ class FakeTokenizer:+ def count_messages(self, messages) -> int: # noqa: ANN001+ return tokenizer_count++ monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))+ monkeypatch.setitem(+ sys.modules,+ "headroom.tokenizers",+ SimpleNamespace(get_tokenizer=lambda model: FakeTokenizer()),+ )+ monkeypatch.setitem(+ sys.modules,+ "headroom.utils",+ SimpleNamespace(extract_user_query=lambda messages: "query"),+ )+++async def test_compress_batch_jsonl_without_optimization_handles_invalid_lines(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(monkeypatch, tokenizer_count=12)+ handler = DummyBatchHandler()+ content = "\n".join(+ [+ json.dumps(+ {"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}}+ ),+ json.dumps({"body": {"model": "gpt-4o", "messages": []}}),+ "not-json",+ ]+ )++ lines, stats = await handler._compress_batch_jsonl(content, "req-1")++ assert len(lines) == 3+ assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hi"+ assert lines[2] == "not-json"+ assert stats == {+ "total_requests": 3,+ "total_original_tokens": 12,+ "total_compressed_tokens": 12,+ "total_tokens_saved": 0,+ "savings_percent": 0.0,+ "errors": 1,+ }+++async def test_compress_batch_jsonl_uses_pipeline_and_ccr_injection(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(+ monkeypatch,+ injector_result=(+ [{"role": "system", "content": "compressed"}],+ [{"name": "retrieval"}],+ True,+ ),+ )+ handler = DummyBatchHandler()+ handler.config.optimize = True+ handler.config.ccr_inject_tool = True+ handler.openai_pipeline = SimpleNamespace(+ apply=lambda **kwargs: SimpleNamespace(+ messages=[{"role": "assistant", "content": "short"}],+ tokens_before=100,+ tokens_after=40,+ )+ )++ lines, stats = await handler._compress_batch_jsonl(+ json.dumps(+ {+ "body": {+ "model": "gpt-4o-mini",+ "messages": [{"role": "user", "content": "hello"}],+ "tools": [{"name": "existing"}],+ }+ }+ ),+ "req-2",+ )++ body = json.loads(lines[0])["body"]+ assert body["messages"] == [{"role": "system", "content": "compressed"}]+ assert body["tools"] == [{"name": "retrieval"}]+ assert stats["total_tokens_saved"] == 60+ assert stats["savings_percent"] == 60.0+++async def test_compress_batch_jsonl_falls_back_when_pipeline_raises(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(monkeypatch, tokenizer_count=33)+ handler = DummyBatchHandler()+ handler.config.optimize = True+ handler.openai_pipeline = SimpleNamespace(+ apply=lambda **kwargs: (_ for _ in ()).throw(RuntimeError("boom"))+ )++ lines, stats = await handler._compress_batch_jsonl(+ json.dumps({"body": {"messages": [{"role": "user", "content": "hello"}]}}),+ "req-3",+ )++ assert json.loads(lines[0])["body"]["messages"][0]["content"] == "hello"+ assert stats["total_original_tokens"] == 33+ assert stats["total_compressed_tokens"] == 33+++async def test_batch_passthrough_forwards_request_and_strips_response_headers() -> None:+ handler = DummyBatchHandler()+ handler.http_client.post_response = FakeResponse(+ content=b'{"ok":true}',+ headers={"content-encoding": "gzip", "content-length": "20", "x-kept": "1"},+ )++ response = await handler._batch_passthrough(+ FakeRequest(+ '{"input_file_id":"file-1"}', headers={"host": "example", "content-length": "10"}+ ),+ {"input_file_id": "file-1"},+ )++ assert response.status_code == 200+ assert dict(response.headers)["x-kept"] == "1"+ assert "content-encoding" not in dict(response.headers)+ assert handler.http_client.posts[0]["url"] == "https://openai.example/v1/batches"+++async def test_handle_batch_create_validates_json_and_required_fields(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ handler = DummyBatchHandler()++ async def raise_bad_json(request): # noqa: ANN001+ raise ValueError("bad json")++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", raise_bad_json)++ bad = await handler.handle_batch_create(FakeRequest("{}"))+ assert bad.status_code == 400+ assert bad.body.decode().find("invalid_json") > 0++ async def missing_file_payload(request): # noqa: ANN001+ return {"endpoint": "/v1/chat/completions"}++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_file_payload)+ missing_file = await handler.handle_batch_create(FakeRequest("{}"))+ assert missing_file.status_code == 400+ assert missing_file.body.decode().find("input_file_id is required") > 0++ async def missing_endpoint_payload(request): # noqa: ANN001+ return {"input_file_id": "file-1"}++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", missing_endpoint_payload)+ missing_endpoint = await handler.handle_batch_create(FakeRequest("{}"))+ assert missing_endpoint.status_code == 400+ assert missing_endpoint.body.decode().find("endpoint is required") > 0+++async def test_handle_batch_create_passthrough_and_download_failure(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ handler = DummyBatchHandler()+ passthrough_response = SimpleNamespace(marker="passthrough")++ async def fake_passthrough(request, body): # noqa: ANN001+ return passthrough_response++ monkeypatch.setattr(handler, "_batch_passthrough", fake_passthrough)++ async def passthrough_payload(request): # noqa: ANN001+ return {"input_file_id": "file-1", "endpoint": "/v1/responses"}++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", passthrough_payload)+ assert await handler.handle_batch_create(FakeRequest("{}")) is passthrough_response++ async def download_missing_payload(request): # noqa: ANN001+ return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}++ async def missing_download(file_id, headers): # noqa: ANN001+ return None++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", download_missing_payload)+ monkeypatch.setattr(handler, "_download_openai_file", missing_download)+ missing = await handler.handle_batch_create(FakeRequest("{}"))+ assert missing.status_code == 404+ assert missing.body.decode().find("file_not_found") > 0+++async def test_handle_batch_create_handles_empty_upload_failure_and_success(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ handler = DummyBatchHandler()++ async def request_payload(request): # noqa: ANN001+ return {+ "input_file_id": "file-1",+ "endpoint": "/v1/chat/completions",+ "completion_window": "12h",+ "metadata": {"source": "test"},+ }++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)++ async def fake_download(file_id, headers): # noqa: ANN001+ return "downloaded"++ monkeypatch.setattr(handler, "_download_openai_file", fake_download)++ async def empty_compress(content, request_id): # noqa: ANN001+ return [], {+ "total_requests": 0,+ "total_original_tokens": 0,+ "total_compressed_tokens": 0,+ "total_tokens_saved": 0,+ "savings_percent": 0.0,+ "errors": 0,+ }++ monkeypatch.setattr(handler, "_compress_batch_jsonl", empty_compress)+ empty = await handler.handle_batch_create(FakeRequest("{}"))+ assert empty.status_code == 400+ assert empty.body.decode().find("empty_file") > 0++ async def compressed(content, request_id): # noqa: ANN001+ return ['{"body":{}}'], {+ "total_requests": 1,+ "total_original_tokens": 20,+ "total_compressed_tokens": 10,+ "total_tokens_saved": 10,+ "savings_percent": 50.0,+ "errors": 0,+ }++ monkeypatch.setattr(handler, "_compress_batch_jsonl", compressed)++ async def upload_failed_file(content, filename, headers): # noqa: ANN001+ return None++ monkeypatch.setattr(handler, "_upload_openai_file", upload_failed_file)+ upload_failed = await handler.handle_batch_create(FakeRequest("{}"))+ assert upload_failed.status_code == 500+ assert upload_failed.body.decode().find("upload_failed") > 0++ handler.http_client.post_response = FakeResponse(+ content=b'{"id":"batch_123","object":"batch"}',+ headers={"content-encoding": "gzip", "content-length": "12", "x-openai": "1"},+ )++ async def upload_success(content, filename, headers): # noqa: ANN001+ return "file-compressed"++ monkeypatch.setattr(handler, "_upload_openai_file", upload_success)+ success = await handler.handle_batch_create(+ FakeRequest(+ "{}", headers={"host": "proxy", "content-length": "4", "authorization": "Bearer test"}+ )+ )++ assert success.status_code == 200+ success_headers = dict(success.headers)+ assert success_headers["x-headroom-tokens-saved"] == "10"+ assert success_headers["x-headroom-savings-percent"] == "50.0"+ assert success_headers["x-openai"] == "1"+ sent_body = handler.http_client.posts[-1]["json"]+ assert sent_body["metadata"]["headroom_compressed"] == "true"+ assert sent_body["metadata"]["headroom_original_file_id"] == "file-1"+ assert handler.metrics.record_calls[-1]["provider"] == "openai"+++async def test_handle_batch_create_records_failure_on_exception(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ handler = DummyBatchHandler()++ async def request_payload(request): # noqa: ANN001+ return {"input_file_id": "file-1", "endpoint": "/v1/chat/completions"}++ async def boom(file_id, headers): # noqa: ANN001+ raise RuntimeError("boom")++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", request_payload)+ monkeypatch.setattr(handler, "_download_openai_file", boom)++ response = await handler.handle_batch_create(FakeRequest("{}"))++ assert response.status_code == 500+ assert handler.metrics.failed_calls == [{"provider": "batch"}]+++async def test_download_and_upload_openai_file_helpers() -> None:+ handler = DummyBatchHandler()+ handler.http_client.get_response = FakeResponse(status_code=200, text="jsonl-content")+ downloaded = await handler._download_openai_file("file-1", {"authorization": "Bearer token"})+ assert downloaded == "jsonl-content"+ assert handler.http_client.gets[0]["url"] == "https://openai.example/v1/files/file-1/content"++ handler.http_client.get_response = FakeResponse(status_code=404, text="missing")+ assert await handler._download_openai_file("file-2", {}) is None++ handler.http_client.post_response = FakeResponse(+ status_code=200,+ json_data={"id": "file-uploaded"},+ headers={"content-type": "application/json"},+ )+ file_id = await handler._upload_openai_file(+ '{"body":{}}',+ "compressed.jsonl",+ {"authorization": "Bearer token", "content-type": "application/json"},+ )+ assert file_id == "file-uploaded"+ post_call = handler.http_client.posts[-1]+ assert post_call["headers"] == {"authorization": "Bearer token"}+ assert post_call["files"]["file"][0] == "compressed.jsonl"++ handler.http_client.post_response = FakeResponse(status_code=500, text="fail")+ assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None+ handler.http_client.raise_post = RuntimeError("network")+ assert await handler._upload_openai_file("{}", "bad.jsonl", {}) is None+++async def test_store_google_batch_context_persists_transformed_requests(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ stored_contexts: list[object] = []++ class FakeBatchContext:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ self.kwargs = kwargs+ self.requests: list[object] = []++ def add_request(self, request) -> None: # noqa: ANN001+ self.requests.append(request)++ class FakeBatchRequestContext:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ self.kwargs = kwargs++ class FakeStore:+ async def store(self, context) -> None: # noqa: ANN001+ stored_contexts.append(context)++ monkeypatch.setitem(+ sys.modules,+ "headroom.ccr",+ SimpleNamespace(+ BatchContext=FakeBatchContext,+ BatchRequestContext=FakeBatchRequestContext,+ get_batch_context_store=lambda: FakeStore(),+ ),+ )++ handler = DummyBatchHandler()+ await handler._store_google_batch_context(+ "batches/123",+ [+ {+ "metadata": {"key": "req-1"},+ "request": {+ "contents": [{"parts": [{"text": "hello"}]}],+ "systemInstruction": {"parts": [{"text": "system"}]},+ "tools": [{"name": "tool"}],+ },+ }+ ],+ "gemini-2.0",+ "api-key",+ )++ context = stored_contexts[0]+ assert context.kwargs["batch_id"] == "batches/123"+ assert context.requests[0].kwargs["custom_id"] == "req-1"+ assert context.requests[0].kwargs["messages"] == [{"role": "user", "content": "hello"}]+ assert context.requests[0].kwargs["system_instruction"] == "system"+++async def test_handle_google_batch_results_passes_through_early_exit_cases(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ class FakeStore:+ async def get(self, batch_name): # noqa: ANN001+ return None++ monkeypatch.setitem(+ sys.modules,+ "headroom.ccr",+ SimpleNamespace(+ BatchResultProcessor=lambda http_client: None,+ get_batch_context_store=lambda: FakeStore(),+ ),+ )++ handler = DummyBatchHandler()+ request = FakeRequest(+ "{}", headers={"x-goog-api-key": "secret"}, method="GET", path="/v1beta/batches/b1"+ )++ handler.http_client.get_response = FakeResponse(+ status_code=500, content=b"bad", headers={"x-upstream": "1"}+ )+ error_response = await handler.handle_google_batch_results(request, "batches/b1")+ assert error_response.status_code == 500+ assert dict(error_response.headers)["x-upstream"] == "1"++ class BadJsonResponse(FakeResponse):+ def json(self): # noqa: ANN201+ raise json.JSONDecodeError("bad", "x", 0)++ handler.http_client.get_response = BadJsonResponse(+ status_code=200, content=b"plain", headers={"x-upstream": "2"}+ )+ non_json = await handler.handle_google_batch_results(request, "batches/b1")+ assert non_json.status_code == 200+ assert dict(non_json.headers)["x-upstream"] == "2"++ handler.http_client.get_response = FakeResponse(+ status_code=200,+ content=b"{}",+ json_data={"metadata": {"state": "RUNNING"}},+ )+ running = await handler.handle_google_batch_results(request, "batches/b1")+ assert running.status_code == 200++ handler.http_client.get_response = FakeResponse(+ status_code=200,+ content=b"{}",+ json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": []}},+ )+ no_results = await handler.handle_google_batch_results(request, "batches/b1")+ assert no_results.status_code == 200++ handler.http_client.get_response = FakeResponse(+ status_code=200,+ content=b"{}",+ json_data={"metadata": {"state": "SUCCEEDED"}, "response": {"responses": [{"id": 1}]}},+ )+ handler.config.ccr_inject_tool = False+ no_ccr = await handler.handle_google_batch_results(request, "batches/b1")+ assert no_ccr.status_code == 200+ assert "key=secret" in handler.http_client.gets[-1]["url"]+++async def test_handle_google_batch_results_processes_completed_results(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ processed_calls: list[tuple[str, list[object], str]] = []++ class FakeProcessed:+ def __init__(+ self, result, custom_id: str, was_processed: bool, continuation_rounds: int+ ) -> None: # noqa: ANN001+ self.result = result+ self.custom_id = custom_id+ self.was_processed = was_processed+ self.continuation_rounds = continuation_rounds++ class FakeProcessor:+ def __init__(self, http_client) -> None: # noqa: ANN001+ self.http_client = http_client++ async def process_results(self, batch_name, results, provider): # noqa: ANN001+ processed_calls.append((batch_name, results, provider))+ return [+ FakeProcessed({"id": "processed"}, "req-1", True, 2),+ FakeProcessed({"id": "unchanged"}, "req-2", False, 0),+ ]++ class FakeStore:+ async def get(self, batch_name): # noqa: ANN001+ return SimpleNamespace(batch_name=batch_name)++ monkeypatch.setitem(+ sys.modules,+ "headroom.ccr",+ SimpleNamespace(+ BatchResultProcessor=FakeProcessor,+ get_batch_context_store=lambda: FakeStore(),+ ),+ )++ handler = DummyBatchHandler()+ handler.config.ccr_inject_tool = True+ handler.http_client.get_response = FakeResponse(+ status_code=200,+ content=b"{}",+ json_data={+ "metadata": {"state": "SUCCEEDED"},+ "response": {"responses": [{"id": "raw-1"}, {"id": "raw-2"}]},+ },+ )++ response = await handler.handle_google_batch_results(+ FakeRequest("{}", method="GET", path="/v1beta/batches/b1"),+ "batches/b1",+ )++ payload = json.loads(response.body)+ assert payload["response"]["responses"] == [{"id": "processed"}, {"id": "unchanged"}]+ assert processed_calls == [("batches/b1", [{"id": "raw-1"}, {"id": "raw-2"}], "google")]+ assert handler.metrics.record_calls[-1]["model"] == "batch:ccr-processed"+++async def test_google_batch_passthrough_helpers_forward_and_track_metrics() -> None:+ handler = DummyBatchHandler()+ handler.http_client.post_response = FakeResponse(+ content=b'{"ok":true}',+ headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},+ )+ handler.http_client.post_response = FakeResponse(+ content=b'{"ok":true}',+ headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "1"},+ )++ passthrough = await handler._google_batch_passthrough(+ FakeRequest(+ "body", headers={"host": "proxy", "content-length": "4", "x-goog-api-key": "secret"}+ ),+ "gemini-pro",+ {"batch": {}},+ )+ assert passthrough.status_code == 200+ assert dict(passthrough.headers)["x-kept"] == "1"+ assert "key=secret" in handler.http_client.posts[-1]["url"]+ assert handler.metrics.record_calls[-1]["model"] == "passthrough:batch:gemini-pro"++ handler.http_client.get_response = FakeResponse(+ content=b'{"state":"ok"}',+ headers={"content-encoding": "gzip", "content-length": "10", "x-kept": "2"},+ )+ response = await handler.handle_google_batch_passthrough(+ FakeRequest(+ "ping",+ headers={"host": "proxy", "x-goog-api-key": "secret"},+ method="DELETE",+ path="/v1beta/batches/b1",+ query="alt=json",+ ),+ "b1",+ )+ assert response.status_code == 200+ assert dict(response.headers)["x-kept"] == "2"+ get_call = handler.http_client.requests[-1]+ assert get_call["url"] == "https://gemini.example/v1beta/batches/b1?alt=json&key=secret"+ assert handler.metrics.record_calls[-1]["model"] == "passthrough:batches"+++async def test_handle_google_batch_create_validates_and_passthroughs(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(monkeypatch)+ handler = DummyBatchHandler()++ too_large = await handler.handle_google_batch_create(+ FakeRequest("{}", headers={"content-length": str(200 * 1024 * 1024)}),+ "gemini-pro",+ )+ assert too_large.status_code == 413++ async def bad_json(request): # noqa: ANN001+ raise ValueError("bad json")++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", bad_json)+ invalid = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")+ assert invalid.status_code == 400++ passthrough_response = SimpleNamespace(kind="passthrough")++ async def fake_google_passthrough(request, model, body=None): # noqa: ANN001+ return passthrough_response++ async def no_inline(request): # noqa: ANN001+ return {"batch": {"input_config": {"requests": {"requests": []}}}}++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", no_inline)+ monkeypatch.setattr(handler, "_google_batch_passthrough", fake_google_passthrough)+ assert (+ await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")+ is passthrough_response+ )+++async def test_handle_google_batch_create_success_and_failure_paths(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(monkeypatch)+ handler = DummyBatchHandler()+ handler.config.optimize = True+ handler.config.ccr_inject_tool = True+ handler.openai_pipeline = SimpleNamespace(+ apply=lambda **kwargs: SimpleNamespace(+ messages=[{"role": "user", "content": "compressed"}],+ timing={"compress": 1.2},+ tokens_before=100,+ tokens_after=40,+ )+ )++ class FakeInjector:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ pass++ def process_request(self, messages, tools): # noqa: ANN001, ANN201+ return (+ messages + [{"role": "system", "content": "retrieval"}],+ [{"name": "retrieval"}],+ True,+ )++ monkeypatch.setitem(sys.modules, "headroom.ccr", SimpleNamespace(CCRToolInjector=FakeInjector))++ stored: list[tuple[str, list[dict[str, object]], str, str | None]] = []++ async def fake_store(batch_name, requests_list, model, api_key): # noqa: ANN001+ stored.append((batch_name, requests_list, model, api_key))++ async def fake_retry(method, url, headers, body): # noqa: ANN001+ return FakeResponse(+ status_code=200,+ content=b'{"name":"batches/123"}',+ headers={"content-encoding": "gzip", "content-length": "10", "x-upstream": "1"},+ json_data={"name": "batches/123"},+ )++ async def good_payload(request): # noqa: ANN001+ return {+ "batch": {+ "input_config": {+ "requests": {+ "requests": [+ {+ "request": {+ "contents": [{"parts": [{"text": "hello"}]}],+ "tools": [{"functionDeclarations": [{"name": "existing"}]}],+ },+ "metadata": {"key": "req-1"},+ }+ ]+ }+ }+ }+ }++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", good_payload)+ monkeypatch.setattr(handler, "_retry_request", fake_retry)+ monkeypatch.setattr(handler, "_store_google_batch_context", fake_store)++ response = await handler.handle_google_batch_create(+ FakeRequest("{}", headers={"x-goog-api-key": "secret"}),+ "gemini-pro",+ )+ assert response.status_code == 200+ assert dict(response.headers)["x-upstream"] == "1"+ assert handler.metrics.record_calls[-1]["provider"] == "google"+ assert handler.metrics.record_calls[-1]["tokens_saved"] == 60+ assert stored[0][0] == "batches/123"+ assert stored[0][2:] == ("gemini-pro", "secret")+ assert stored[0][1][0]["metadata"] == {"key": "req-1"}++ async def broken_retry(method, url, headers, body): # noqa: ANN001+ raise RuntimeError("forward failed")++ monkeypatch.setattr(handler, "_retry_request", broken_retry)+ failed = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")+ assert failed.status_code == 500+++async def test_handle_google_batch_create_covers_passthrough_revert_and_store_failures(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(+ monkeypatch, injector_result=([{"role": "user", "content": "kept"}], None, False)+ )+ handler = DummyBatchHandler()+ handler.config.optimize = True+ handler.config.ccr_inject_tool = True++ pipeline_calls: list[dict[str, object]] = []+ handler.openai_pipeline = SimpleNamespace(+ apply=lambda **kwargs: pipeline_calls.append(kwargs)+ or SimpleNamespace(+ messages=[{"role": "user", "content": "inflated"}],+ timing={},+ tokens_before=40,+ tokens_after=80,+ )+ )++ def fake_to_messages(contents, system_instruction): # noqa: ANN001, ANN201+ if contents and "inlineData" in contents[0]["parts"][0]:+ return ([{"role": "user", "content": "binary"}], [0])+ return ([{"role": "user", "content": "compress"}], [])++ def fake_to_gemini(messages): # noqa: ANN001, ANN201+ return ([{"parts": [{"text": "new"}]}], {"parts": [{"text": "sys"}]})++ async def payload(request): # noqa: ANN001+ return {+ "batch": {+ "input_config": {+ "requests": {+ "requests": [+ {"request": {"contents": []}, "metadata": {"key": "empty"}},+ {+ "request": {"contents": [{"parts": [{"inlineData": "x"}]}]},+ "metadata": {"key": "preserved"},+ },+ {+ "request": {+ "contents": [{"parts": [{"text": "hello"}]}],+ "tools": [+ {"other": True},+ {"functionDeclarations": [{"name": "existing"}]},+ ],+ },+ "metadata": {"key": "optimized"},+ },+ ]+ }+ }+ }+ }++ seen_bodies: list[dict[str, object]] = []++ async def retry(method, url, headers, body): # noqa: ANN001+ seen_bodies.append(body)+ return FakeResponse(status_code=200, content=b"{}", json_data={"name": "batches/123"})++ async def broken_store(batch_name, requests_list, model, api_key): # noqa: ANN001+ raise RuntimeError("store failed")++ monkeypatch.setattr("headroom.proxy.helpers._read_request_json", payload)+ monkeypatch.setattr(handler, "_gemini_contents_to_messages", fake_to_messages)+ monkeypatch.setattr(handler, "_messages_to_gemini_contents", fake_to_gemini)+ monkeypatch.setattr(handler, "_retry_request", retry)+ monkeypatch.setattr(handler, "_store_google_batch_context", broken_store)++ response = await handler.handle_google_batch_create(FakeRequest("{}"), "gemini-pro")+ assert response.status_code == 200+ assert len(pipeline_calls) == 1+ assert handler.metrics.record_calls[-1]["tokens_saved"] == 0+ assert (+ seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][0]["metadata"]["key"]+ == "empty"+ )+ optimized = seen_bodies[0]["batch"]["input_config"]["requests"]["requests"][2]["request"]+ assert optimized["contents"][0] == {"parts": [{"text": "new"}]}+ assert optimized["systemInstruction"] == {"parts": [{"text": "sys"}]}+++async def test_google_batch_passthrough_without_body_and_query_variants() -> None:+ handler = DummyBatchHandler()+ handler.http_client.post_response = FakeResponse(content=b"ok", headers={"x-upstream": "1"})++ response = await handler._google_batch_passthrough(+ FakeRequest("raw-body", headers={"host": "proxy"}, method="POST"),+ "gemini-pro",+ )+ assert response.status_code == 200+ assert handler.http_client.posts[-1]["content"] == b"raw-body"++ handler.http_client.get_response = FakeResponse(content=b"{}", headers={"x-upstream": "2"})+ passthrough = await handler.handle_google_batch_passthrough(+ FakeRequest(+ "{}",+ headers={"host": "proxy", "x-goog-api-key": "secret"},+ method="GET",+ path="/v1beta/batches/b1",+ ),+ "b1",+ )+ assert passthrough.status_code == 200+ assert (+ handler.http_client.requests[-1]["url"]+ == "https://gemini.example/v1beta/batches/b1?key=secret"+ )+++async def test_batch_helper_methods_and_openai_file_error_branches() -> None:+ handler = DummyBatchHandler()+ marker = object()++ async def fake_passthrough(request, base_url): # noqa: ANN001+ return marker++ handler.handle_passthrough = fake_passthrough+ request = FakeRequest("{}")+ assert await handler.handle_batch_list(request) is marker+ assert await handler.handle_batch_get(request, "b1") is marker+ assert await handler.handle_batch_cancel(request, "b1") is marker++ handler.http_client.raise_get = RuntimeError("download boom")+ assert await handler._download_openai_file("file-1", {}) is None++ handler.http_client.raise_get = None+ handler.http_client.post_response = FakeResponse(status_code=200, json_data={})+ assert await handler._upload_openai_file("{}", "missing-id.jsonl", {}) is None+++async def test_store_google_batch_context_without_system_text() -> None:+ stored_contexts: list[object] = []++ class FakeBatchContext:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ self.kwargs = kwargs+ self.requests: list[object] = []++ def add_request(self, request) -> None: # noqa: ANN001+ self.requests.append(request)++ class FakeBatchRequestContext:+ def __init__(self, **kwargs) -> None: # noqa: ANN003+ self.kwargs = kwargs++ class FakeStore:+ async def store(self, context) -> None: # noqa: ANN001+ stored_contexts.append(context)++ handler = DummyBatchHandler()+ sys.modules["headroom.ccr"] = SimpleNamespace(+ BatchContext=FakeBatchContext,+ BatchRequestContext=FakeBatchRequestContext,+ get_batch_context_store=lambda: FakeStore(),+ )++ await handler._store_google_batch_context(+ "batches/456",+ [+ {+ "request": {+ "contents": [{"parts": [{"text": "hello"}]}],+ "systemInstruction": {"parts": ["bad"]},+ }+ }+ ],+ "gemini-2.0",+ None,+ )++ context = stored_contexts[0]+ assert context.kwargs["api_key"] is None+ assert context.requests[0].kwargs["custom_id"] == ""+ assert context.requests[0].kwargs["system_instruction"] is None+++async def test_compress_batch_jsonl_skips_blank_lines_and_preserves_tools_when_not_injected(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ install_batch_support_modules(+ monkeypatch,+ injector_result=([{"role": "assistant", "content": "short"}], [{"name": "orig"}], False),+ )+ handler = DummyBatchHandler()+ handler.config.optimize = True+ handler.config.ccr_inject_tool = True+ handler.openai_pipeline = SimpleNamespace(+ apply=lambda **kwargs: SimpleNamespace(+ messages=[{"role": "assistant", "content": "short"}],+ tokens_before=50,+ tokens_after=10,+ )+ )++ lines, stats = await handler._compress_batch_jsonl(+ "\n"+ + json.dumps(+ {+ "body": {+ "model": "gpt-4o",+ "messages": [{"role": "user", "content": "hello"}],+ "tools": [{"name": "orig"}],+ }+ }+ )+ + "\n",+ "req-extra",+ )++ assert len(lines) == 1+ body = json.loads(lines[0])["body"]+ assert body["tools"] == [{"name": "orig"}]+ assert stats["total_requests"] == 1+ assert stats["errors"] == 0diff --git a/tests/test_relevance_extra.py b/tests/test_relevance_extra.pyindex e5c705a5e..1dc25a29f 100644--- a/tests/test_relevance_extra.py+++ b/tests/test_relevance_extra.py@@ -1,181 +1,181 @@-from __future__ import annotations--import builtins-from dataclasses import dataclass-from types import SimpleNamespace--import pytest--import headroom.relevance as relevance_mod-from headroom.relevance import (- BM25Scorer,- EmbeddingScorer,- HybridScorer,- create_scorer,- embedding,- hybrid,-)-from headroom.relevance.base import RelevanceScore, RelevanceScorer, default_batch_score---@dataclass-class DummyRelevanceScorer(RelevanceScorer):- def score(self, item: str, context: str) -> RelevanceScore:- return RelevanceScore(score=0.4, reason=f"{item}:{context}")-- def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:- return [RelevanceScore(score=0.2, reason=context) for _ in items]---def test_base_default_batch_and_abstract_methods() -> None:- scorer = DummyRelevanceScorer()- batch = default_batch_score(scorer, ["a", "b"], "ctx")- assert [item.reason for item in batch] == ["a:ctx", "b:ctx"]-- assert RelevanceScorer.score(scorer, "a", "ctx") is None- assert RelevanceScorer.score_batch(scorer, ["a"], "ctx") is None- assert RelevanceScorer.is_available() is True---def test_create_scorer_embedding_unavailable_branch(monkeypatch) -> None:- monkeypatch.setattr(- relevance_mod.EmbeddingScorer, "is_available", classmethod(lambda cls: False)- )- with pytest.raises(RuntimeError, match="sentence-transformers"):- create_scorer("embedding")---def test_bm25_internal_paths_and_non_normalized_mode() -> None:- scorer = BM25Scorer(normalize_score=False)- assert scorer._tokenize("") == []- assert scorer._compute_idf("x", doc_count=1, doc_freq=0) == 0.0- assert scorer._compute_idf("x", doc_count=1, doc_freq=1) > 0- assert scorer._bm25_score([], ["a"]) == (0.0, [])- assert scorer._bm25_score(["a"], []) == (0.0, [])-- no_match = scorer.score("hello world", "missing")- assert no_match.reason == "BM25: no term matches"-- one_match = scorer.score("find alice", "alice")- assert one_match.reason == "BM25: matched 'alice'"- assert one_match.score > 0-- many_match = scorer.score("alpha beta gamma delta", "alpha beta gamma delta")- assert many_match.reason.startswith("BM25: matched 4 terms")-- batch = scorer.score_batch(["alpha", "alpha beta"], "alpha beta")- assert [item.reason for item in batch] == ["BM25: 1 terms", "BM25: 2 terms"]---def test_embedding_numpy_and_model_error_paths(monkeypatch) -> None:- embedding._numpy = None- real_import = builtins.__import__-- def fake_import(name, globals=None, locals=None, fromlist=(), level=0):- if name == "numpy":- raise ImportError("missing")- return real_import(name, globals, locals, fromlist, level)-- monkeypatch.setattr(builtins, "__import__", fake_import)- with pytest.raises(ImportError, match="numpy is required"):- embedding._get_numpy()-- monkeypatch.setattr(builtins, "__import__", real_import)- fake_np = SimpleNamespace(- linalg=SimpleNamespace(norm=lambda value: 0 if value == [0, 0] else 1),- dot=lambda a, b: -1,- )- monkeypatch.setattr(embedding, "_numpy", fake_np)- assert embedding._cosine_similarity([0, 0], [1, 0]) == 0.0- assert embedding._cosine_similarity([1, 0], [0, 1]) == 0.0-- monkeypatch.setattr(EmbeddingScorer, "is_available", classmethod(lambda cls: False))- with pytest.raises(RuntimeError, match="requires sentence-transformers"):- EmbeddingScorer()._get_model()---def test_embedding_score_empty_and_batch_shortcuts() -> None:- scorer = EmbeddingScorer()- assert scorer.score("", "ctx").reason == "Embedding: empty input"- assert scorer.score("item", "").reason == "Embedding: empty input"- assert scorer.score_batch([], "ctx") == []- assert scorer.score_batch(["item"], "")[0].reason == "Embedding: empty context"---def test_embedding_score_and_batch_with_fake_model(monkeypatch) -> None:- scorer = EmbeddingScorer()- monkeypatch.setattr(- scorer,- "_encode",- lambda texts: [[1.0, 0.0], [0.5, 0.5]]- if len(texts) == 2- else [[1.0, 0.0], [0.0, 1.0], [1.0, 0.0]],- )- monkeypatch.setattr(- embedding, "_cosine_similarity", lambda a, b: 0.75 if a == [1.0, 0.0] else 0.25- )-- single = scorer.score("item", "ctx")- assert single.score == 0.75- assert single.reason == "Embedding: semantic similarity 0.75"-- batch = scorer.score_batch(["first", "second"], "ctx")- assert [item.score for item in batch] == [0.75, 0.25]- assert [item.reason for item in batch] == ["Embedding: 0.75", "Embedding: 0.25"]---def test_hybrid_constructor_alpha_variants_and_single_score_paths(monkeypatch) -> None:- bm25_result = RelevanceScore(score=0.1, reason="bm25", matched_terms=["term"])- emb_result = RelevanceScore(score=0.9, reason="emb", matched_terms=[])-- class FakeBM25:- def score(self, item: str, context: str) -> RelevanceScore:- return bm25_result-- def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:- return [bm25_result for _ in items]-- class FakeEmbedding:- def score(self, item: str, context: str) -> RelevanceScore:- return emb_result-- def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:- return [emb_result for _ in items]-- scorer = HybridScorer(- alpha=0.4, adaptive=True, bm25_scorer=FakeBM25(), embedding_scorer=FakeEmbedding()- )- assert scorer.has_embedding_support() is True- assert scorer._compute_alpha("find id 1234") == 0.65- assert scorer._compute_alpha("find host api.example.com") == 0.6- assert scorer._compute_alpha("find email [email protected]") == 0.6-- single = scorer.score("item", "show me errors")- assert single.score == pytest.approx(0.58)- assert "Hybrid (α=0.40): BM25=0.10, Semantic=0.90" == single.reason-- batch = scorer.score_batch(["a", "b"], "show me errors")- assert len(batch) == 2- assert batch[0].reason == "Hybrid (α=0.40): BM25=0.10, Emb=0.90"---def test_hybrid_fallback_and_empty_batch(monkeypatch) -> None:- scorer = HybridScorer(bm25_scorer=BM25Scorer())- scorer._embedding_available = False- scorer.embedding = None-- empty = scorer.score_batch([], "ctx")- assert empty == []-- boosted = scorer.score('{"id":"123","name":"alice"}', "alice")- assert boosted.score >= 0.3- assert "BM25 only, boosted" in boosted.reason-- boosted_batch = scorer.score_batch(['{"id":"123"}', '{"id":"456"}'], "123 456")- assert all("BM25 only, boosted" in item.reason for item in boosted_batch)---def test_hybrid_auto_fallback_when_embeddings_unavailable(monkeypatch) -> None:- monkeypatch.setattr(hybrid.EmbeddingScorer, "is_available", classmethod(lambda cls: False))- scorer = HybridScorer()- assert scorer.has_embedding_support() is False+from __future__ import annotations++import builtins+from dataclasses import dataclass+from types import SimpleNamespace++import pytest++import headroom.relevance as relevance_mod+from headroom.relevance import (+ BM25Scorer,+ EmbeddingScorer,+ HybridScorer,+ create_scorer,+ embedding,+ hybrid,+)+from headroom.relevance.base import RelevanceScore, RelevanceScorer, default_batch_score+++@dataclass+class DummyRelevanceScorer(RelevanceScorer):+ def score(self, item: str, context: str) -> RelevanceScore:+ return RelevanceScore(score=0.4, reason=f"{item}:{context}")++ def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:+ return [RelevanceScore(score=0.2, reason=context) for _ in items]+++def test_base_default_batch_and_abstract_methods() -> None:+ scorer = DummyRelevanceScorer()+ batch = default_batch_score(scorer, ["a", "b"], "ctx")+ assert [item.reason for item in batch] == ["a:ctx", "b:ctx"]++ assert RelevanceScorer.score(scorer, "a", "ctx") is None+ assert RelevanceScorer.score_batch(scorer, ["a"], "ctx") is None+ assert RelevanceScorer.is_available() is True+++def test_create_scorer_embedding_unavailable_branch(monkeypatch) -> None:+ monkeypatch.setattr(+ relevance_mod.EmbeddingScorer, "is_available", classmethod(lambda cls: False)+ )+ with pytest.raises(RuntimeError, match="sentence-transformers"):+ create_scorer("embedding")+++def test_bm25_internal_paths_and_non_normalized_mode() -> None:+ scorer = BM25Scorer(normalize_score=False)+ assert scorer._tokenize("") == []+ assert scorer._compute_idf("x", doc_count=1, doc_freq=0) == 0.0+ assert scorer._compute_idf("x", doc_count=1, doc_freq=1) > 0+ assert scorer._bm25_score([], ["a"]) == (0.0, [])+ assert scorer._bm25_score(["a"], []) == (0.0, [])++ no_match = scorer.score("hello world", "missing")+ assert no_match.reason == "BM25: no term matches"++ one_match = scorer.score("find alice", "alice")+ assert one_match.reason == "BM25: matched 'alice'"+ assert one_match.score > 0++ many_match = scorer.score("alpha beta gamma delta", "alpha beta gamma delta")+ assert many_match.reason.startswith("BM25: matched 4 terms")++ batch = scorer.score_batch(["alpha", "alpha beta"], "alpha beta")+ assert [item.reason for item in batch] == ["BM25: 1 terms", "BM25: 2 terms"]+++def test_embedding_numpy_and_model_error_paths(monkeypatch) -> None:+ embedding._numpy = None+ real_import = builtins.__import__++ def fake_import(name, globals=None, locals=None, fromlist=(), level=0):+ if name == "numpy":+ raise ImportError("missing")+ return real_import(name, globals, locals, fromlist, level)++ monkeypatch.setattr(builtins, "__import__", fake_import)+ with pytest.raises(ImportError, match="numpy is required"):+ embedding._get_numpy()++ monkeypatch.setattr(builtins, "__import__", real_import)+ fake_np = SimpleNamespace(+ linalg=SimpleNamespace(norm=lambda value: 0 if value == [0, 0] else 1),+ dot=lambda a, b: -1,+ )+ monkeypatch.setattr(embedding, "_numpy", fake_np)+ assert embedding._cosine_similarity([0, 0], [1, 0]) == 0.0+ assert embedding._cosine_similarity([1, 0], [0, 1]) == 0.0++ monkeypatch.setattr(EmbeddingScorer, "is_available", classmethod(lambda cls: False))+ with pytest.raises(RuntimeError, match="requires sentence-transformers"):+ EmbeddingScorer()._get_model()+++def test_embedding_score_empty_and_batch_shortcuts() -> None:+ scorer = EmbeddingScorer()+ assert scorer.score("", "ctx").reason == "Embedding: empty input"+ assert scorer.score("item", "").reason == "Embedding: empty input"+ assert scorer.score_batch([], "ctx") == []+ assert scorer.score_batch(["item"], "")[0].reason == "Embedding: empty context"+++def test_embedding_score_and_batch_with_fake_model(monkeypatch) -> None:+ scorer = EmbeddingScorer()+ monkeypatch.setattr(+ scorer,+ "_encode",+ lambda texts: [[1.0, 0.0], [0.5, 0.5]]+ if len(texts) == 2+ else [[1.0, 0.0], [0.0, 1.0], [1.0, 0.0]],+ )+ monkeypatch.setattr(+ embedding, "_cosine_similarity", lambda a, b: 0.75 if a == [1.0, 0.0] else 0.25+ )++ single = scorer.score("item", "ctx")+ assert single.score == 0.75+ assert single.reason == "Embedding: semantic similarity 0.75"++ batch = scorer.score_batch(["first", "second"], "ctx")+ assert [item.score for item in batch] == [0.75, 0.25]+ assert [item.reason for item in batch] == ["Embedding: 0.75", "Embedding: 0.25"]+++def test_hybrid_constructor_alpha_variants_and_single_score_paths(monkeypatch) -> None:+ bm25_result = RelevanceScore(score=0.1, reason="bm25", matched_terms=["term"])+ emb_result = RelevanceScore(score=0.9, reason="emb", matched_terms=[])++ class FakeBM25:+ def score(self, item: str, context: str) -> RelevanceScore:+ return bm25_result++ def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:+ return [bm25_result for _ in items]++ class FakeEmbedding:+ def score(self, item: str, context: str) -> RelevanceScore:+ return emb_result++ def score_batch(self, items: list[str], context: str) -> list[RelevanceScore]:+ return [emb_result for _ in items]++ scorer = HybridScorer(+ alpha=0.4, adaptive=True, bm25_scorer=FakeBM25(), embedding_scorer=FakeEmbedding()+ )+ assert scorer.has_embedding_support() is True+ assert scorer._compute_alpha("find id 1234") == 0.65+ assert scorer._compute_alpha("find host api.example.com") == 0.6+ assert scorer._compute_alpha("find email [email protected]") == 0.6++ single = scorer.score("item", "show me errors")+ assert single.score == pytest.approx(0.58)+ assert "Hybrid (α=0.40): BM25=0.10, Semantic=0.90" == single.reason++ batch = scorer.score_batch(["a", "b"], "show me errors")+ assert len(batch) == 2+ assert batch[0].reason == "Hybrid (α=0.40): BM25=0.10, Emb=0.90"+++def test_hybrid_fallback_and_empty_batch(monkeypatch) -> None:+ scorer = HybridScorer(bm25_scorer=BM25Scorer())+ scorer._embedding_available = False+ scorer.embedding = None++ empty = scorer.score_batch([], "ctx")+ assert empty == []++ boosted = scorer.score('{"id":"123","name":"alice"}', "alice")+ assert boosted.score >= 0.3+ assert "BM25 only, boosted" in boosted.reason++ boosted_batch = scorer.score_batch(['{"id":"123"}', '{"id":"456"}'], "123 456")+ assert all("BM25 only, boosted" in item.reason for item in boosted_batch)+++def test_hybrid_auto_fallback_when_embeddings_unavailable(monkeypatch) -> None:+ monkeypatch.setattr(hybrid.EmbeddingScorer, "is_available", classmethod(lambda cls: False))+ scorer = HybridScorer()+ assert scorer.has_embedding_support() is Falsediff --git a/tests/test_storage_backends.py b/tests/test_storage_backends.pyindex 7800fb133..0521c8ade 100644--- a/tests/test_storage_backends.py+++ b/tests/test_storage_backends.py@@ -1,294 +1,294 @@-from __future__ import annotations--from dataclasses import dataclass-from datetime import datetime, timedelta-from pathlib import Path-from types import SimpleNamespace--from headroom.config import RequestMetrics-from headroom.storage import JSONLStorage, SQLiteStorage, Storage, create_storage---def _metrics(- request_id: str,- timestamp: datetime,- model: str = "gpt-4o",- mode: str = "audit",- before: int = 100,- after: int = 80,-) -> RequestMetrics:- return RequestMetrics(- request_id=request_id,- timestamp=timestamp,- model=model,- stream=False,- mode=mode,- tokens_input_before=before,- tokens_input_after=after,- tokens_output=25,- block_breakdown={"system": 10},- waste_signals={"json_bloat": 3},- stable_prefix_hash="prefix",- cache_alignment_score=75.0,- cached_tokens=12,- transforms_applied=["compress"],- tool_units_dropped=1,- turns_dropped=2,- messages_hash="messages",- error=None,- )---@dataclass-class DummyStorage(Storage):- closed: bool = False-- def save(self, metrics: RequestMetrics) -> None:- return None-- def get(self, request_id: str) -> RequestMetrics | None:- return None-- def query(self, **kwargs) -> list[RequestMetrics]:- return []-- def count(self, **kwargs) -> int:- return 0-- def iter_all(self):- return iter(())-- def get_summary_stats(self, **kwargs) -> dict[str, int]:- return {}-- def close(self) -> None:- self.closed = True---def test_storage_base_context_manager_calls_close() -> None:- storage = DummyStorage()- with storage as managed:- assert managed is storage- assert storage.closed is False- assert storage.closed is True-- assert Storage.save(storage, _metrics("x", datetime(2026, 4, 23, 12, 0, 0))) is None- assert Storage.get(storage, "x") is None- assert Storage.query(storage) is None- assert Storage.count(storage) is None- assert Storage.iter_all(storage) is None- assert Storage.get_summary_stats(storage) is None- assert Storage.close(storage) is None---def test_create_storage_builtin_entrypoint_and_fallback(monkeypatch, tmp_path: Path) -> None:- sqlite_storage = create_storage(f"sqlite://{tmp_path}\\metrics.db")- jsonl_storage = create_storage(f"jsonl://{tmp_path}\\metrics.jsonl")- assert isinstance(sqlite_storage, SQLiteStorage)- assert isinstance(jsonl_storage, JSONLStorage)- sqlite_storage.close()- jsonl_storage.close()-- absolute_sqlite = create_storage("sqlite:///tmp/demo.db")- absolute_jsonl = create_storage("jsonl:///tmp/demo.jsonl")- assert isinstance(absolute_sqlite, SQLiteStorage)- assert isinstance(absolute_jsonl, JSONLStorage)- absolute_sqlite.close()- absolute_jsonl.close()-- created = DummyStorage()-- class FakeEntryPoint:- name = "custom"-- def load(self):- return lambda store_url: created-- monkeypatch.setattr(- "importlib.metadata.entry_points",- lambda group: [FakeEntryPoint()] if group == "headroom.storage_backend" else [],- )- assert create_storage("custom://memory") is created-- monkeypatch.setattr(- "importlib.metadata.entry_points", lambda group: (_ for _ in ()).throw(RuntimeError("boom"))- )- created_fallback: list[str] = []-- class FakeSQLiteStorage:- def __init__(self, db_path: str) -> None:- created_fallback.append(db_path)-- def close(self) -> None:- return None-- monkeypatch.setattr("headroom.storage.SQLiteStorage", FakeSQLiteStorage)- fallback = create_storage("custom://fallback.db")- assert isinstance(fallback, FakeSQLiteStorage)- assert created_fallback == ["custom://fallback.db"]- fallback.close()-- monkeypatch.setattr(- "importlib.metadata.entry_points",- lambda group: [SimpleNamespace(name="other", load=lambda: (lambda url: created))],- )- missing_ep = create_storage("custom://missing.db")- assert isinstance(missing_ep, FakeSQLiteStorage)- assert created_fallback == ["custom://fallback.db", "custom://missing.db"]- missing_ep.close()-- plain = create_storage("metrics.db")- assert isinstance(plain, FakeSQLiteStorage)- assert created_fallback == ["custom://fallback.db", "custom://missing.db", "metrics.db"]- plain.close()---def test_jsonl_storage_round_trip_query_count_and_summary(tmp_path: Path) -> None:- storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))- now = datetime(2026, 4, 23, 12, 0, 0)- first = _metrics("one", now - timedelta(hours=2), mode="audit", before=120, after=100)- second = _metrics(- "two", now - timedelta(hours=1), model="claude", mode="optimize", before=90, after=30- )- third = _metrics("three", now, mode="audit", before=60, after=50)-- storage.save(first)- storage.save(second)- storage.save(third)-- assert storage.get("two") == second- assert storage.get("missing") is None-- results = storage.query(start_time=now - timedelta(hours=1, minutes=30), offset=1, limit=1)- assert [item.request_id for item in results] == ["three"]- assert storage.query(model="claude")[0].request_id == "two"- assert storage.query(mode="optimize")[0].request_id == "two"- assert storage.query(end_time=now - timedelta(hours=1, minutes=30))[0].request_id == "one"- assert storage.count(mode="audit") == 2- assert storage.count(end_time=now - timedelta(hours=1, minutes=30)) == 1- assert storage.count(start_time=now + timedelta(days=1)) == 0-- summary = storage.get_summary_stats(start_time=now - timedelta(hours=3), end_time=now)- assert summary == {- "total_requests": 3,- "total_tokens_before": 270,- "total_tokens_after": 180,- "total_tokens_saved": 90,- "avg_tokens_saved": 30.0,- "avg_cache_alignment": 75.0,- "audit_count": 2,- "optimize_count": 1,- }-- storage.close()---def test_jsonl_storage_handles_missing_file_malformed_lines_and_defaults(tmp_path: Path) -> None:- path = tmp_path / "events.jsonl"- storage = JSONLStorage(str(path))- path.unlink()- assert list(storage.iter_all()) == []-- path.write_text(- "\n".join(- [- "",- "not-json",- '{"id":"x","timestamp":"2026-04-23T12:00:00Z","model":"gpt-4o","stream":true,"mode":"simulate","tokens_input_before":5,"tokens_input_after":3}',- ]- )- )- loaded = list(storage.iter_all())- assert len(loaded) == 1- assert loaded[0].request_id == "x"- assert loaded[0].tokens_output is None- assert loaded[0].block_breakdown == {}- assert loaded[0].waste_signals == {}- assert loaded[0].stable_prefix_hash == ""- assert loaded[0].cache_alignment_score == 0.0- assert loaded[0].transforms_applied == []- assert loaded[0].tool_units_dropped == 0- assert loaded[0].turns_dropped == 0- assert loaded[0].messages_hash == ""- assert loaded[0].error is None---def test_sqlite_storage_round_trip_filters_summary_and_defaults(tmp_path: Path) -> None:- storage = SQLiteStorage(str(tmp_path / "metrics.db"))- now = datetime(2026, 4, 23, 12, 0, 0)- first = _metrics("one", now - timedelta(hours=2), mode="audit", before=100, after=70)- second = _metrics(- "two", now - timedelta(hours=1), model="claude", mode="optimize", before=90, after=20- )- third = _metrics("three", now, before=50, after=50)- third.stable_prefix_hash = ""- third.cache_alignment_score = 0.0- third.cached_tokens = None- third.transforms_applied = []- third.tool_units_dropped = 0- third.turns_dropped = 0- third.messages_hash = ""-- storage.save(first)- storage.save(second)- storage.save(third)- replacement = _metrics("one", now + timedelta(minutes=1), before=111, after=11)- storage.save(replacement)-- assert storage.get("one") == replacement- assert storage.get("missing") is None-- results = storage.query(start_time=now - timedelta(hours=2), end_time=now, limit=2, offset=1)- assert [item.request_id for item in results] == ["two"]- assert storage.query(model="claude")[0].request_id == "two"- assert storage.query(mode="optimize")[0].request_id == "two"- assert storage.count(mode="audit") == 2- assert storage.count(start_time=now - timedelta(hours=1, minutes=30), end_time=now) == 2- assert storage.count(model="missing") == 0- assert [item.request_id for item in storage.iter_all()] == ["two", "three", "one"]-- summary = storage.get_summary_stats(- start_time=now - timedelta(hours=3), end_time=now + timedelta(hours=1)- )- assert summary == {- "total_requests": 3,- "total_tokens_before": 251,- "total_tokens_after": 81,- "total_tokens_saved": 170,- "avg_tokens_saved": 56.666666666666664,- "avg_cache_alignment": 50.0,- "audit_count": 2,- "optimize_count": 1,- }-- empty = storage.get_summary_stats(start_time=now + timedelta(days=1))- assert empty == {- "total_requests": 0,- "total_tokens_before": 0,- "total_tokens_after": 0,- "total_tokens_saved": 0,- "avg_tokens_saved": 0,- "avg_cache_alignment": 0,- "audit_count": 0,- "optimize_count": 0,- }-- storage.close()- assert storage._conn is None---def test_sqlite_storage_get_conn_reuses_connection_and_create_storage_entrypoint(- monkeypatch, tmp_path: Path-) -> None:- storage = SQLiteStorage(str(tmp_path / "metrics.db"))- first = storage._get_conn()- second = storage._get_conn()- assert first is second- storage.close()-- created = DummyStorage()- monkeypatch.setattr(- "importlib.metadata.entry_points",- lambda group: [SimpleNamespace(name="custom", load=lambda: (lambda url: created))],- )- assert create_storage("custom://db") is created+from __future__ import annotations++from dataclasses import dataclass+from datetime import datetime, timedelta+from pathlib import Path+from types import SimpleNamespace++from headroom.config import RequestMetrics+from headroom.storage import JSONLStorage, SQLiteStorage, Storage, create_storage+++def _metrics(+ request_id: str,+ timestamp: datetime,+ model: str = "gpt-4o",+ mode: str = "audit",+ before: int = 100,+ after: int = 80,+) -> RequestMetrics:+ return RequestMetrics(+ request_id=request_id,+ timestamp=timestamp,+ model=model,+ stream=False,+ mode=mode,+ tokens_input_before=before,+ tokens_input_after=after,+ tokens_output=25,+ block_breakdown={"system": 10},+ waste_signals={"json_bloat": 3},+ stable_prefix_hash="prefix",+ cache_alignment_score=75.0,+ cached_tokens=12,+ transforms_applied=["compress"],+ tool_units_dropped=1,+ turns_dropped=2,+ messages_hash="messages",+ error=None,+ )+++@dataclass+class DummyStorage(Storage):+ closed: bool = False++ def save(self, metrics: RequestMetrics) -> None:+ return None++ def get(self, request_id: str) -> RequestMetrics | None:+ return None++ def query(self, **kwargs) -> list[RequestMetrics]:+ return []++ def count(self, **kwargs) -> int:+ return 0++ def iter_all(self):+ return iter(())++ def get_summary_stats(self, **kwargs) -> dict[str, int]:+ return {}++ def close(self) -> None:+ self.closed = True+++def test_storage_base_context_manager_calls_close() -> None:+ storage = DummyStorage()+ with storage as managed:+ assert managed is storage+ assert storage.closed is False+ assert storage.closed is True++ assert Storage.save(storage, _metrics("x", datetime(2026, 4, 23, 12, 0, 0))) is None+ assert Storage.get(storage, "x") is None+ assert Storage.query(storage) is None+ assert Storage.count(storage) is None+ assert Storage.iter_all(storage) is None+ assert Storage.get_summary_stats(storage) is None+ assert Storage.close(storage) is None+++def test_create_storage_builtin_entrypoint_and_fallback(monkeypatch, tmp_path: Path) -> None:+ sqlite_storage = create_storage(f"sqlite://{tmp_path}\\metrics.db")+ jsonl_storage = create_storage(f"jsonl://{tmp_path}\\metrics.jsonl")+ assert isinstance(sqlite_storage, SQLiteStorage)+ assert isinstance(jsonl_storage, JSONLStorage)+ sqlite_storage.close()+ jsonl_storage.close()++ absolute_sqlite = create_storage("sqlite:///tmp/demo.db")+ absolute_jsonl = create_storage("jsonl:///tmp/demo.jsonl")+ assert isinstance(absolute_sqlite, SQLiteStorage)+ assert isinstance(absolute_jsonl, JSONLStorage)+ absolute_sqlite.close()+ absolute_jsonl.close()++ created = DummyStorage()++ class FakeEntryPoint:+ name = "custom"++ def load(self):+ return lambda store_url: created++ monkeypatch.setattr(+ "importlib.metadata.entry_points",+ lambda group: [FakeEntryPoint()] if group == "headroom.storage_backend" else [],+ )+ assert create_storage("custom://memory") is created++ monkeypatch.setattr(+ "importlib.metadata.entry_points", lambda group: (_ for _ in ()).throw(RuntimeError("boom"))+ )+ created_fallback: list[str] = []++ class FakeSQLiteStorage:+ def __init__(self, db_path: str) -> None:+ created_fallback.append(db_path)++ def close(self) -> None:+ return None++ monkeypatch.setattr("headroom.storage.SQLiteStorage", FakeSQLiteStorage)+ fallback = create_storage("custom://fallback.db")+ assert isinstance(fallback, FakeSQLiteStorage)+ assert created_fallback == ["custom://fallback.db"]+ fallback.close()++ monkeypatch.setattr(+ "importlib.metadata.entry_points",+ lambda group: [SimpleNamespace(name="other", load=lambda: (lambda url: created))],+ )+ missing_ep = create_storage("custom://missing.db")+ assert isinstance(missing_ep, FakeSQLiteStorage)+ assert created_fallback == ["custom://fallback.db", "custom://missing.db"]+ missing_ep.close()++ plain = create_storage("metrics.db")+ assert isinstance(plain, FakeSQLiteStorage)+ assert created_fallback == ["custom://fallback.db", "custom://missing.db", "metrics.db"]+ plain.close()+++def test_jsonl_storage_round_trip_query_count_and_summary(tmp_path: Path) -> None:+ storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ now = datetime(2026, 4, 23, 12, 0, 0)+ first = _metrics("one", now - timedelta(hours=2), mode="audit", before=120, after=100)+ second = _metrics(+ "two", now - timedelta(hours=1), model="claude", mode="optimize", before=90, after=30+ )+ third = _metrics("three", now, mode="audit", before=60, after=50)++ storage.save(first)+ storage.save(second)+ storage.save(third)++ assert storage.get("two") == second+ assert storage.get("missing") is None++ results = storage.query(start_time=now - timedelta(hours=1, minutes=30), offset=1, limit=1)+ assert [item.request_id for item in results] == ["three"]+ assert storage.query(model="claude")[0].request_id == "two"+ assert storage.query(mode="optimize")[0].request_id == "two"+ assert storage.query(end_time=now - timedelta(hours=1, minutes=30))[0].request_id == "one"+ assert storage.count(mode="audit") == 2+ assert storage.count(end_time=now - timedelta(hours=1, minutes=30)) == 1+ assert storage.count(start_time=now + timedelta(days=1)) == 0++ summary = storage.get_summary_stats(start_time=now - timedelta(hours=3), end_time=now)+ assert summary == {+ "total_requests": 3,+ "total_tokens_before": 270,+ "total_tokens_after": 180,+ "total_tokens_saved": 90,+ "avg_tokens_saved": 30.0,+ "avg_cache_alignment": 75.0,+ "audit_count": 2,+ "optimize_count": 1,+ }++ storage.close()+++def test_jsonl_storage_handles_missing_file_malformed_lines_and_defaults(tmp_path: Path) -> None:+ path = tmp_path / "events.jsonl"+ storage = JSONLStorage(str(path))+ path.unlink()+ assert list(storage.iter_all()) == []++ path.write_text(+ "\n".join(+ [+ "",+ "not-json",+ '{"id":"x","timestamp":"2026-04-23T12:00:00Z","model":"gpt-4o","stream":true,"mode":"simulate","tokens_input_before":5,"tokens_input_after":3}',+ ]+ )+ )+ loaded = list(storage.iter_all())+ assert len(loaded) == 1+ assert loaded[0].request_id == "x"+ assert loaded[0].tokens_output is None+ assert loaded[0].block_breakdown == {}+ assert loaded[0].waste_signals == {}+ assert loaded[0].stable_prefix_hash == ""+ assert loaded[0].cache_alignment_score == 0.0+ assert loaded[0].transforms_applied == []+ assert loaded[0].tool_units_dropped == 0+ assert loaded[0].turns_dropped == 0+ assert loaded[0].messages_hash == ""+ assert loaded[0].error is None+++def test_sqlite_storage_round_trip_filters_summary_and_defaults(tmp_path: Path) -> None:+ storage = SQLiteStorage(str(tmp_path / "metrics.db"))+ now = datetime(2026, 4, 23, 12, 0, 0)+ first = _metrics("one", now - timedelta(hours=2), mode="audit", before=100, after=70)+ second = _metrics(+ "two", now - timedelta(hours=1), model="claude", mode="optimize", before=90, after=20+ )+ third = _metrics("three", now, before=50, after=50)+ third.stable_prefix_hash = ""+ third.cache_alignment_score = 0.0+ third.cached_tokens = None+ third.transforms_applied = []+ third.tool_units_dropped = 0+ third.turns_dropped = 0+ third.messages_hash = ""++ storage.save(first)+ storage.save(second)+ storage.save(third)+ replacement = _metrics("one", now + timedelta(minutes=1), before=111, after=11)+ storage.save(replacement)++ assert storage.get("one") == replacement+ assert storage.get("missing") is None++ results = storage.query(start_time=now - timedelta(hours=2), end_time=now, limit=2, offset=1)+ assert [item.request_id for item in results] == ["two"]+ assert storage.query(model="claude")[0].request_id == "two"+ assert storage.query(mode="optimize")[0].request_id == "two"+ assert storage.count(mode="audit") == 2+ assert storage.count(start_time=now - timedelta(hours=1, minutes=30), end_time=now) == 2+ assert storage.count(model="missing") == 0+ assert [item.request_id for item in storage.iter_all()] == ["two", "three", "one"]++ summary = storage.get_summary_stats(+ start_time=now - timedelta(hours=3), end_time=now + timedelta(hours=1)+ )+ assert summary == {+ "total_requests": 3,+ "total_tokens_before": 251,+ "total_tokens_after": 81,+ "total_tokens_saved": 170,+ "avg_tokens_saved": 56.666666666666664,+ "avg_cache_alignment": 50.0,+ "audit_count": 2,+ "optimize_count": 1,+ }++ empty = storage.get_summary_stats(start_time=now + timedelta(days=1))+ assert empty == {+ "total_requests": 0,+ "total_tokens_before": 0,+ "total_tokens_after": 0,+ "total_tokens_saved": 0,+ "avg_tokens_saved": 0,+ "avg_cache_alignment": 0,+ "audit_count": 0,+ "optimize_count": 0,+ }++ storage.close()+ assert storage._conn is None+++def test_sqlite_storage_get_conn_reuses_connection_and_create_storage_entrypoint(+ monkeypatch, tmp_path: Path+) -> None:+ storage = SQLiteStorage(str(tmp_path / "metrics.db"))+ first = storage._get_conn()+ second = storage._get_conn()+ assert first is second+ storage.close()++ created = DummyStorage()+ monkeypatch.setattr(+ "importlib.metadata.entry_points",+ lambda group: [SimpleNamespace(name="custom", load=lambda: (lambda url: created))],+ )+ assert create_storage("custom://db") is createddiff --git a/tests/test_transforms_log_compressor.py b/tests/test_transforms_log_compressor.pynew file mode 100644index 000000000..b273bb846--- /dev/null+++ b/tests/test_transforms_log_compressor.py@@ -0,0 +1,183 @@+from __future__ import annotations++from types import SimpleNamespace++import pytest++from headroom.transforms.log_compressor import (+ LogCompressionResult,+ LogCompressor,+ LogCompressorConfig,+ LogFormat,+ LogLevel,+ LogLine,+)+++def test_detect_parse_and_score_log_lines() -> None:+ compressor = LogCompressor(LogCompressorConfig(stack_trace_max_lines=2))+ pytest_lines = [+ "============================= test session starts =============================",+ "collected 2 items",+ "ERROR critical failure",+ "Traceback (most recent call last)",+ ' File "app.py", line 10',+ "",+ "2 failed, 1 warning",+ ]+ assert compressor._detect_format(pytest_lines) is LogFormat.PYTEST+ assert compressor._detect_format(["npm ERR! missing script"]) is LogFormat.NPM+ assert compressor._detect_format(["Compiling app", "warning: check this"]) is LogFormat.CARGO+ assert (+ compressor._detect_format(["PASS src/app.test.js", "Test Suites: 1 failed"])+ is LogFormat.JEST+ )+ assert compressor._detect_format(["make: *** fail", "gcc -o app app.c"]) is LogFormat.MAKE+ assert compressor._detect_format(["unclassified line"]) is LogFormat.GENERIC++ parsed = compressor._parse_lines(pytest_lines)+ assert parsed[0].is_summary is True+ assert parsed[2].level is LogLevel.ERROR+ assert parsed[3].is_stack_trace is True+ assert parsed[4].is_stack_trace is True+ assert parsed[6].is_summary is True+ assert compressor._score_line(LogLine(1, "warn", level=LogLevel.WARN)) == 0.5+ assert (+ compressor._score_line(+ LogLine(2, "error summary", level=LogLevel.ERROR, is_stack_trace=True, is_summary=True)+ )+ == 1.0+ )+++def test_select_dedupe_add_context_and_format_output(monkeypatch: pytest.MonkeyPatch) -> None:+ compressor = LogCompressor(+ LogCompressorConfig(+ max_errors=2,+ max_warnings=1,+ error_context_lines=1,+ max_stack_traces=1,+ stack_trace_max_lines=2,+ )+ )+ monkeypatch.setitem(+ __import__("sys").modules,+ "headroom.transforms.adaptive_sizer",+ SimpleNamespace(compute_optimal_k=lambda items, **kwargs: 6),+ )+ log_lines = [+ LogLine(0, "info line", level=LogLevel.INFO, score=0.1),+ LogLine(1, "ERROR first", level=LogLevel.ERROR, score=1.0),+ LogLine(2, "context after first", level=LogLevel.UNKNOWN, score=0.1),+ LogLine(3, "WARNING /tmp/a/123 issue", level=LogLevel.WARN, score=0.5),+ LogLine(4, "WARNING /tmp/b/999 issue", level=LogLevel.WARN, score=0.5),+ LogLine(5, "FAIL final", level=LogLevel.FAIL, score=1.0),+ LogLine(6, "Traceback (most recent call last)", is_stack_trace=True, score=0.4),+ LogLine(7, ' File "app.py", line 2', is_stack_trace=True, score=0.4),+ LogLine(8, "1 failed, 1 warning", is_summary=True, score=0.5),+ ]+ selected = compressor._select_lines(log_lines)+ assert [line.line_number for line in selected] == [1, 3, 4, 5, 6, 8]++ assert compressor._select_with_first_last(log_lines[:2], max_count=5) == log_lines[:2]+ many_errors = [+ LogLine(10, "first", level=LogLevel.ERROR, score=0.1),+ LogLine(11, "mid", level=LogLevel.ERROR, score=0.9),+ LogLine(12, "last", level=LogLevel.ERROR, score=0.2),+ ]+ trimmed = compressor._select_with_first_last(many_errors, max_count=2)+ assert trimmed == [many_errors[0], many_errors[2]]+ deduped = compressor._dedupe_similar(log_lines[3:5])+ assert len(deduped) == 1++ output, stats = compressor._format_output(selected, log_lines)+ assert stats == {+ "errors": 1,+ "fails": 1,+ "warnings": 2,+ "info": 1,+ "total": 9,+ "selected": 6,+ }+ assert output.endswith("[3 lines omitted: 1 ERROR, 1 FAIL, 2 WARN, 1 INFO]")+++def test_log_compressor_compress_and_ccr_paths(monkeypatch: pytest.MonkeyPatch) -> None:+ compressor = LogCompressor(LogCompressorConfig(enable_ccr=True, min_lines_for_ccr=3))+ short = compressor.compress("a\nb")+ assert short.format_detected is LogFormat.GENERIC+ assert short.compression_ratio == 1.0++ monkeypatch.setattr(compressor, "_detect_format", lambda lines: LogFormat.NPM)+ parsed = [LogLine(0, "npm ERR! boom", level=LogLevel.ERROR, score=1.0)]+ monkeypatch.setattr(compressor, "_parse_lines", lambda lines: parsed)+ monkeypatch.setattr(compressor, "_select_lines", lambda log_lines, bias=1.0: parsed)+ monkeypatch.setattr(+ compressor,+ "_format_output",+ lambda selected, all_lines: (+ "tiny",+ {"errors": 1, "fails": 0, "warnings": 0, "info": 0, "total": 3, "selected": 1},+ ),+ )+ monkeypatch.setattr(compressor, "_store_in_ccr", lambda original, compressed, count: "deadbeef")+ result = compressor.compress(+ "x\ny\nz\nvery verbose fourth line to improve compression ratio math"+ )+ assert result.format_detected is LogFormat.NPM+ assert result.cache_key == "deadbeef"+ assert result.stats["errors"] == 1+ assert result.compressed.endswith("[4 lines compressed to 1. Retrieve more: hash=deadbeef]")++ monkeypatch.setattr(compressor, "_store_in_ccr", lambda original, compressed, count: None)+ no_cache = compressor.compress(+ "x\ny\nz\nvery verbose fourth line to improve compression ratio math"+ )+ assert no_cache.cache_key is None+ assert no_cache.compressed == "tiny"++ monkeypatch.setattr(+ compressor,+ "_format_output",+ lambda selected, all_lines: (+ "this output is intentionally much longer than the original content",+ {"errors": 1, "fails": 0, "warnings": 0, "info": 0, "total": 4, "selected": 1},+ ),+ )+ high_ratio = compressor.compress("x\ny\nz\nw")+ assert high_ratio.cache_key is None+++def test_store_in_ccr_and_result_properties(monkeypatch: pytest.MonkeyPatch) -> None:+ compressor = LogCompressor()+ monkeypatch.setitem(+ __import__("sys").modules,+ "headroom.cache.compression_store",+ SimpleNamespace(+ get_compression_store=lambda: SimpleNamespace(+ store=lambda original, compressed, original_item_count=0: "stored-log"+ )+ ),+ )+ assert compressor._store_in_ccr("orig", "comp", 10) == "stored-log"++ def broken_store():+ raise RuntimeError("boom")++ monkeypatch.setitem(+ __import__("sys").modules,+ "headroom.cache.compression_store",+ SimpleNamespace(get_compression_store=broken_store),+ )+ assert compressor._store_in_ccr("orig", "comp", 10) is None++ result = LogCompressionResult(+ compressed="small",+ original="this is a substantially longer log body",+ original_line_count=20,+ compressed_line_count=5,+ format_detected=LogFormat.GENERIC,+ compression_ratio=0.25,+ )+ assert result.tokens_saved_estimate > 0+ assert result.lines_omitted == 15