Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(storage): page JSONL queries after sorting (#3872)
charlieanna91237cae7077
diff --git a/headroom/storage/jsonl.py b/headroom/storage/jsonl.pyindex b44ef2535..0e7ec78d2 100644--- a/headroom/storage/jsonl.py+++ b/headroom/storage/jsonl.py@@ -2,6 +2,7 @@from __future__ import annotations+import heapqimport jsonfrom collections.abc import Iteratorfrom datetime import datetime@@ -101,35 +102,51 @@ class JSONLStorage(Storage):limit: int = 100,offset: int = 0,) -> list[RequestMetrics]:- """Query metrics with filters."""- results: list[RequestMetrics] = []- skipped = 0+ """Query metrics with filters.- for metrics in self.iter_all():- # Apply filters+ Matches are ordered by timestamp descending, then request_id+ descending (the order SQLiteStorage.query() applies), before the+ ``[offset : offset + limit]`` page is returned. Selection is bounded+ to that window instead of materializing the full match set.++ Args:+ start_time: Filter by timestamp >= start_time.+ end_time: Filter by timestamp <= end_time.+ model: Filter by model name.+ mode: Filter by mode (audit/optimize).+ limit: Maximum results to return; must be non-negative.+ offset: Number of results to skip; must be non-negative.++ Returns:+ List of matching RequestMetrics for the requested page.++ Raises:+ ValueError: If limit or offset is negative.+ """+ if limit < 0:+ raise ValueError(f"limit must be non-negative, got {limit}")+ if offset < 0:+ raise ValueError(f"offset must be non-negative, got {offset}")+ if limit == 0:+ return []++ def _matches(metrics: RequestMetrics) -> bool:if start_time is not None and metrics.timestamp < start_time:- continue+ return Falseif end_time is not None and metrics.timestamp > end_time:- continue+ return Falseif model is not None and metrics.model != model:- continue+ return Falseif mode is not None and metrics.mode != mode:- continue-- # Handle offset- if skipped < offset:- skipped += 1- continue-- results.append(metrics)+ return False+ return True- # Handle limit- if len(results) >= limit:- break+ filtered = (metrics for metrics in self.iter_all() if _matches(metrics))- # Sort by timestamp descending- results.sort(key=lambda m: m.timestamp, reverse=True)- return results+ # Keep only the top offset + limit matches under the shared+ # timestamp DESC, request_id DESC order, then drop the offset prefix.+ top = heapq.nlargest(offset + limit, filtered, key=lambda m: (m.timestamp, m.request_id))+ return top[offset : offset + limit]def count(self,diff --git a/headroom/storage/sqlite.py b/headroom/storage/sqlite.pyindex a719694b4..c5106edc6 100644--- a/headroom/storage/sqlite.py+++ b/headroom/storage/sqlite.py@@ -142,7 +142,31 @@ class SQLiteStorage(Storage):limit: int = 100,offset: int = 0,) -> list[RequestMetrics]:- """Query metrics with filters."""+ """Query metrics with filters.++ Matches are ordered by timestamp descending, then request_id+ descending (the order JSONLStorage.query() applies), before the+ ``LIMIT/OFFSET`` page is returned.++ Args:+ start_time: Filter by timestamp >= start_time.+ end_time: Filter by timestamp <= end_time.+ model: Filter by model name.+ mode: Filter by mode (audit/optimize).+ limit: Maximum results to return; must be non-negative.+ offset: Number of results to skip; must be non-negative.++ Returns:+ List of matching RequestMetrics for the requested page.++ Raises:+ ValueError: If limit or offset is negative.+ """+ if limit < 0:+ raise ValueError(f"limit must be non-negative, got {limit}")+ if offset < 0:+ raise ValueError(f"offset must be non-negative, got {offset}")+conn = self._get_conn()cursor = conn.cursor()@@ -162,7 +186,9 @@ class SQLiteStorage(Storage):query += " AND mode = ?"params.append(mode)- query += " ORDER BY timestamp DESC LIMIT ? OFFSET ?"+ # Timestamp DESC, id DESC: the same fully-defined order+ # JSONLStorage.query() applies, so tied timestamps page identically.+ query += " ORDER BY timestamp DESC, id DESC LIMIT ? OFFSET ?"params.extend([limit, offset])cursor.execute(query, params)diff --git a/tests/test_storage/test_jsonl.py b/tests/test_storage/test_jsonl.pynew file mode 100644index 000000000..4d18e8150--- /dev/null+++ b/tests/test_storage/test_jsonl.py@@ -0,0 +1,246 @@+"""Tests for JSONL storage query paging (shared newest-first contract)."""++from datetime import datetime, timedelta++import pytest++from headroom.config import RequestMetrics+from headroom.storage.jsonl import JSONLStorage+from headroom.storage.sqlite import SQLiteStorage+++def _metrics(+ request_id: str,+ timestamp: datetime,+ model: str = "gpt-4o",+ mode: str = "audit",+) -> RequestMetrics:+ return RequestMetrics(+ request_id=request_id,+ timestamp=timestamp,+ model=model,+ stream=False,+ mode=mode,+ tokens_input_before=1000,+ tokens_input_after=800,+ tokens_output=200,+ block_breakdown={"system": 100},+ waste_signals={"json_bloat": 5},+ stable_prefix_hash=f"prefix-{request_id}",+ cache_alignment_score=80.0,+ cached_tokens=50,+ transforms_applied=["SmartCrusher"],+ tool_units_dropped=1,+ turns_dropped=0,+ messages_hash=f"messages-{request_id}",+ error=None,+ )+++def _expected_page(+ rows: list[RequestMetrics],+ *,+ model: str | None = None,+ mode: str | None = None,+ start_time: datetime | None = None,+ end_time: datetime | None = None,+ limit: int = 100,+ offset: int = 0,+) -> list[str]:+ """Reference contract: newest-first ordering, then the [offset:offset+limit] page."""+ filtered = [+ row+ for row in rows+ if (start_time is None or row.timestamp >= start_time)+ and (end_time is None or row.timestamp <= end_time)+ and (model is None or row.model == model)+ and (mode is None or row.mode == mode)+ ]+ ordered = sorted(filtered, key=lambda row: row.timestamp, reverse=True)+ return [row.request_id for row in ordered[offset : offset + limit]]+++def _ids(metrics: list[RequestMetrics]) -> list[str]:+ return [m.request_id for m in metrics]+++def _assert_timestamp_desc(metrics: list[RequestMetrics]) -> None:+ stamps = [m.timestamp for m in metrics]+ assert stamps == sorted(stamps, reverse=True)+++class TestJSONLQueryPagingContract:+ """JSONLStorage.query() must page like SQLiteStorage.query() (issue #3822).++ Rows are appended in timestamp order, so paging before ordering returns the+ oldest remaining rows per page. The contract requires the filtered set to be+ ordered newest-first first and only then sliced to [offset:offset+limit].+ """++ @pytest.fixture+ def backends(self, tmp_path):+ jsonl = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ sqlite = SQLiteStorage(str(tmp_path / "metrics.db"))+ base = datetime(2025, 1, 6, 12, 0, 0)+ rows = [+ _metrics(+ f"req-{i:03d}",+ base + timedelta(seconds=i),+ model="claude" if i % 2 == 0 else "gpt-4o",+ mode="audit" if i % 3 == 0 else "optimize",+ )+ for i in range(250)+ ]+ for row in rows:+ jsonl.save(row)+ sqlite.save(row)+ yield jsonl, sqlite, rows+ jsonl.close()+ sqlite.close()++ def test_multi_page_agrees_with_sqlite_and_starts_at_newest(self, backends):+ jsonl, sqlite, rows = backends++ for offset in (0, 5, 10):+ kw = {"limit": 5, "offset": offset}+ jsonl_page = jsonl.query(**kw)+ sqlite_page = sqlite.query(**kw)++ assert _ids(jsonl_page) == _ids(sqlite_page)+ assert _ids(jsonl_page) == _expected_page(rows, **kw)+ _assert_timestamp_desc(jsonl_page)++ first_page = jsonl.query(limit=5, offset=0)+ assert first_page[0].request_id == "req-249"+ assert _ids(first_page) == [+ "req-249",+ "req-248",+ "req-247",+ "req-246",+ "req-245",+ ]++ def test_pages_are_disjoint_and_tile_the_newest_first_order(self, backends):+ jsonl, sqlite, rows = backends++ paged: list[str] = []+ jsonl_pages: list[list[str]] = []+ offset = 0+ while True:+ ids = _ids(jsonl.query(limit=5, offset=offset))+ if not ids:+ break+ jsonl_pages.append(ids)+ assert ids == _ids(sqlite.query(limit=5, offset=offset))+ paged.extend(ids)+ offset += 5++ assert sum(len(page) for page in jsonl_pages) == 250+ for i, page in enumerate(jsonl_pages):+ for other in jsonl_pages[i + 1 :]:+ assert set(page).isdisjoint(other)++ assert paged == _expected_page(rows, limit=250, offset=0)++ def test_filtered_pages_agree_with_sqlite(self, backends):+ jsonl, sqlite, rows = backends++ cases = [+ {"model": "claude", "limit": 5, "offset": 0},+ {"model": "claude", "limit": 5, "offset": 5},+ {"mode": "audit", "limit": 5, "offset": 0},+ {"mode": "audit", "limit": 4, "offset": 2},+ {"model": "claude", "mode": "audit", "limit": 3, "offset": 1},+ {+ "model": "gpt-4o",+ "mode": "optimize",+ "start_time": datetime(2025, 1, 6, 12, 0, 30),+ "limit": 6,+ "offset": 3,+ },+ ]+ for kw in cases:+ jsonl_page = jsonl.query(**kw)+ sqlite_page = sqlite.query(**kw)++ assert _ids(jsonl_page) == _ids(sqlite_page), kw+ assert _ids(jsonl_page) == _expected_page(rows, **kw), kw+ _assert_timestamp_desc(jsonl_page)++ def test_filtered_pages_tile_the_filtered_set(self, backends):+ jsonl, sqlite, rows = backends++ kw = {"mode": "audit"}+ total = jsonl.count(**kw)+ assert total == sqlite.count(**kw) == len(_expected_page(rows, **kw, limit=250))++ tiled: list[str] = []+ offset = 0+ while True:+ page_kw = {**kw, "limit": 7, "offset": offset}+ ids = _ids(jsonl.query(**page_kw))+ if not ids:+ break+ assert ids == _ids(sqlite.query(**page_kw)), page_kw+ tiled.extend(ids)+ offset += 7++ assert tiled == _expected_page(rows, **kw, limit=250, offset=0)++ def test_page_one_is_newest_matching_row_not_newest_overall(self, backends):+ jsonl, sqlite, rows = backends++ # req-249 is newest overall and an audit row (249 % 3 == 0), but the+ # newest optimize row is req-248; a filtered page must start there.+ page = jsonl.query(mode="optimize", limit=5, offset=0)+ assert page[0].request_id == "req-248"+ assert _ids(page) == _ids(sqlite.query(mode="optimize", limit=5, offset=0))++ def test_count_summary_and_iter_all_are_unchanged(self, backends):+ jsonl, sqlite, rows = backends++ assert jsonl.count() == 250+ assert jsonl.count(mode="audit") == sqlite.count(mode="audit")+ assert jsonl.count(model="claude") == sqlite.count(model="claude")+ assert (+ jsonl.count(+ start_time=datetime(2025, 1, 6, 12, 0, 30),+ end_time=datetime(2025, 1, 6, 12, 1, 0),+ )+ == 31+ )++ summary = jsonl.get_summary_stats()+ assert summary["total_requests"] == 250+ assert summary["total_tokens_before"] == 250 * 1000+ assert summary["total_tokens_after"] == 250 * 800+ assert summary["audit_count"] == jsonl.count(mode="audit")+ assert summary["optimize_count"] == jsonl.count(mode="optimize")++ # iter_all() keeps file (append) order: paging fixes must not sort it.+ assert _ids(list(jsonl.iter_all())) == [row.request_id for row in rows]+++class TestJSONLQuerySingleRowRegression:+ """The 3-row scenario from tests/test_storage_backends.py:162 (issue #3822)."""++ def test_offset_window_over_filtered_set_is_newest_first(self, tmp_path):+ jsonl = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ sqlite = SQLiteStorage(str(tmp_path / "metrics.db"))+ now = datetime(2026, 4, 23, 12, 0, 0)+ first = _metrics("one", now - timedelta(hours=2), mode="audit")+ second = _metrics("two", now - timedelta(hours=1), model="claude", mode="optimize")+ third = _metrics("three", now, mode="audit")+ for row in (first, second, third):+ jsonl.save(row)+ sqlite.save(row)++ kw = {+ "start_time": now - timedelta(hours=1, minutes=30),+ "offset": 1,+ "limit": 1,+ }+ assert _ids(jsonl.query(**kw)) == _ids(sqlite.query(**kw)) == ["two"]++ jsonl.close()+ sqlite.close()diff --git a/tests/test_storage_backends.py b/tests/test_storage_backends.pyindex 00860d556..0f35dc882 100644--- a/tests/test_storage_backends.py+++ b/tests/test_storage_backends.py@@ -1,9 +1,14 @@from __future__ import annotations+import weakref+from collections.abc import Callable, Iteratorfrom dataclasses import dataclassfrom datetime import datetime, timedeltafrom pathlib import Pathfrom types import SimpleNamespace+from typing import Any++import pytestfrom headroom.config import RequestMetricsfrom headroom.storage import JSONLStorage, SQLiteStorage, Storage, create_storage@@ -49,16 +54,34 @@ class DummyStorage(Storage):def get(self, request_id: str) -> RequestMetrics | None:return None- def query(self, **kwargs) -> list[RequestMetrics]:+ def query(+ self,+ start_time: datetime | None = None,+ end_time: datetime | None = None,+ model: str | None = None,+ mode: str | None = None,+ limit: int = 100,+ offset: int = 0,+ ) -> list[RequestMetrics]:return []- def count(self, **kwargs) -> int:+ def count(+ self,+ start_time: datetime | None = None,+ end_time: datetime | None = None,+ model: str | None = None,+ mode: str | None = None,+ ) -> int:return 0- def iter_all(self):+ def iter_all(self) -> Iterator[RequestMetrics]:return iter(())- def get_summary_stats(self, **kwargs) -> dict[str, int]:+ def get_summary_stats(+ self,+ start_time: datetime | None = None,+ end_time: datetime | None = None,+ ) -> dict[str, Any]:return {}def close(self) -> None:@@ -81,7 +104,9 @@ def test_storage_base_context_manager_calls_close() -> None:assert Storage.close(storage) is None-def test_create_storage_builtin_entrypoint_and_fallback(monkeypatch, tmp_path: Path) -> None:+def test_create_storage_builtin_entrypoint_and_fallback(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:sqlite_storage = create_storage(f"sqlite://{tmp_path}\\metrics.db")jsonl_storage = create_storage(f"jsonl://{tmp_path}\\metrics.jsonl")assert isinstance(sqlite_storage, SQLiteStorage)@@ -101,7 +126,7 @@ def test_create_storage_builtin_entrypoint_and_fallback(monkeypatch, tmp_path: Pclass FakeEntryPoint:name = "custom"- def load(self):+ def load(self) -> Callable[[str], DummyStorage]:return lambda store_url: createdmonkeypatch.setattr(@@ -160,7 +185,7 @@ def test_jsonl_storage_round_trip_query_count_and_summary(tmp_path: Path) -> Nonassert storage.get("missing") is Noneresults = storage.query(start_time=now - timedelta(hours=1, minutes=30), offset=1, limit=1)- assert [item.request_id for item in results] == ["three"]+ assert [item.request_id for item in results] == ["two"]assert storage.query(model="claude")[0].request_id == "two"assert storage.query(mode="optimize")[0].request_id == "two"assert storage.query(end_time=now - timedelta(hours=1, minutes=30))[0].request_id == "one"@@ -183,6 +208,59 @@ def test_jsonl_storage_round_trip_query_count_and_summary(tmp_path: Path) -> Nonstorage.close()+def test_jsonl_query_paging_matches_sqlite_newest_first(tmp_path: Path) -> None:+ """JSONL query pages must be the newest-first window SQLite returns (issue #3822)."""+ jsonl_storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ sqlite_storage = SQLiteStorage(str(tmp_path / "metrics.db"))+ base = datetime(2026, 4, 23, 12, 0, 0)+ rows = [+ _metrics(+ f"req-{i:03d}",+ base + timedelta(seconds=i),+ model="claude" if i % 2 == 0 else "gpt-4o",+ mode="audit" if i % 3 == 0 else "optimize",+ )+ for i in range(250)+ ]+ for row in rows:+ jsonl_storage.save(row)+ sqlite_storage.save(row)++ def ids(metrics: list[RequestMetrics]) -> list[str]:+ return [item.request_id for item in metrics]++ newest_first = [+ row.request_id for row in sorted(rows, key=lambda item: item.timestamp, reverse=True)+ ]++ for kw in (+ {"limit": 5, "offset": 0},+ {"limit": 5, "offset": 5},+ {"limit": 5, "offset": 10},+ {"model": "claude", "limit": 5, "offset": 0},+ {"model": "claude", "limit": 5, "offset": 5},+ {"mode": "audit", "limit": 4, "offset": 2},+ {"model": "claude", "mode": "audit", "limit": 3, "offset": 1},+ ):+ jsonl_page = jsonl_storage.query(**kw)+ assert ids(jsonl_page) == ids(sqlite_storage.query(**kw)), kw+ stamps = [item.timestamp for item in jsonl_page]+ assert stamps == sorted(stamps, reverse=True), kw++ assert ids(jsonl_storage.query(limit=5)) == newest_first[:5]+ assert ids(jsonl_storage.query(limit=5))[0] == "req-249"++ page1 = ids(jsonl_storage.query(limit=5, offset=0))+ page2 = ids(jsonl_storage.query(limit=5, offset=5))+ page3 = ids(jsonl_storage.query(limit=5, offset=10))+ assert set(page1).isdisjoint(page2)+ assert set(page2).isdisjoint(page3)+ assert set(page1).isdisjoint(page3)++ jsonl_storage.close()+ sqlite_storage.close()++def test_jsonl_storage_handles_missing_file_malformed_lines_and_defaults(tmp_path: Path) -> None:path = tmp_path / "events.jsonl"storage = JSONLStorage(str(path))@@ -278,7 +356,7 @@ def test_sqlite_storage_round_trip_filters_summary_and_defaults(tmp_path: Path)def test_sqlite_storage_get_conn_reuses_connection_and_create_storage_entrypoint(- monkeypatch, tmp_path: Path+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:storage = SQLiteStorage(str(tmp_path / "metrics.db"))first = storage._get_conn()@@ -292,3 +370,140 @@ def test_sqlite_storage_get_conn_reuses_connection_and_create_storage_entrypointlambda group: [SimpleNamespace(name="custom", load=lambda: lambda url: created)],)assert create_storage("custom://db") is created+++def test_query_pagination_contract_validates_and_returns_empty_pages(tmp_path: Path) -> None:+ """Negative limit/offset raise ValueError; zero limit and past-end offsets page empty."""+ jsonl_storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ sqlite_storage = SQLiteStorage(str(tmp_path / "metrics.db"))+ now = datetime(2026, 4, 23, 12, 0, 0)+ rows = [+ _metrics("one", now - timedelta(hours=2)),+ _metrics("two", now - timedelta(hours=1)),+ _metrics("three", now),+ ]+ for row in rows:+ jsonl_storage.save(row)+ sqlite_storage.save(row)++ for storage in (jsonl_storage, sqlite_storage):+ with pytest.raises(ValueError):+ storage.query(limit=-1)+ with pytest.raises(ValueError):+ storage.query(offset=-1)+ with pytest.raises(ValueError):+ storage.query(limit=-5, offset=-5)++ assert storage.query(limit=0) == []+ assert storage.query(limit=0, offset=0) == []+ assert storage.query(limit=0, offset=9) == []+ assert storage.query(limit=5, offset=3) == []+ assert storage.query(limit=5, offset=10) == []+ assert [m.request_id for m in storage.query(limit=2, offset=0)] == ["three", "two"]+ assert [m.request_id for m in storage.query(limit=2, offset=2)] == ["one"]++ jsonl_storage.close()+ sqlite_storage.close()+++def test_query_equal_timestamp_pages_match_across_backends(tmp_path: Path) -> None:+ """Timestamp ties break on request_id DESC and pages agree page-for-page."""+ jsonl_storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ sqlite_storage = SQLiteStorage(str(tmp_path / "metrics.db"))+ base = datetime(2026, 4, 23, 12, 0, 0)++ rows: list[RequestMetrics] = []+ # Rows are appended in a shuffled order on purpose: the contract orders by+ # timestamp DESC then request_id DESC, never by append order.+ for group in (1, 2, 3):+ for suffix in (3, 1, 5, 2, 4):+ rows.append(+ _metrics(+ f"g{group}-{suffix}",+ base + timedelta(minutes=group),+ model="claude" if suffix % 2 == 0 else "gpt-4o",+ )+ )+ for row in rows:+ jsonl_storage.save(row)+ sqlite_storage.save(row)++ expected = [f"g{group}-{suffix}" for group in (3, 2, 1) for suffix in (5, 4, 3, 2, 1)]++ for offset in (0, 4, 8, 12):+ page_expected = expected[offset : offset + 4]+ jsonl_page = jsonl_storage.query(limit=4, offset=offset)+ sqlite_page = sqlite_storage.query(limit=4, offset=offset)+ assert [m.request_id for m in jsonl_page] == page_expected, offset+ assert [m.request_id for m in sqlite_page] == page_expected, offset+ stamps = [m.timestamp for m in jsonl_page]+ assert stamps == sorted(stamps, reverse=True)+ # The same window must be stable across repeated queries.+ assert [m.request_id for m in jsonl_storage.query(limit=4, offset=offset)] == page_expected+ assert [m.request_id for m in sqlite_storage.query(limit=4, offset=offset)] == page_expected++ # Filtered ties must stay consistent too: a page size of 3 splits the g2+ # pair across pages so the tie-breaker decides both sides of the boundary.+ filtered_expected = [f"g{group}-{suffix}" for group in (3, 2, 1) for suffix in (4, 2)]+ for offset in (0, 3):+ page_expected = filtered_expected[offset : offset + 3]+ assert [+ m.request_id for m in jsonl_storage.query(model="claude", limit=3, offset=offset)+ ] == page_expected+ assert [+ m.request_id for m in sqlite_storage.query(model="claude", limit=3, offset=offset)+ ] == page_expected++ jsonl_storage.close()+ sqlite_storage.close()+++def test_jsonl_query_retains_at_most_requested_window(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """query() must select a bounded window instead of materializing every match."""+ storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))+ base = datetime(2026, 4, 23, 12, 0, 0)+ rows = [+ _metrics(+ f"req-{i:03d}",+ base + timedelta(seconds=i),+ model="claude" if i % 2 == 0 else "gpt-4o",+ )+ for i in range(60)+ ]+ for row in rows:+ storage.save(row)++ limit, offset = 5, 10+ window = offset + limit+ matches = [row for row in rows if row.model == "claude"]+ assert len(matches) > window++ original_iter_all = storage.iter_all+ refs: list[weakref.ReferenceType[RequestMetrics]] = []+ kept: list[bool] = []+ retained: list[int] = []++ def tracked_iter_all() -> Iterator[RequestMetrics]:+ for metrics in original_iter_all():+ matched = metrics.model == "claude"+ # Count matching records yielded earlier that the query still+ # holds; the record being handed over is not in refs yet.+ retained.append(sum(1 for keep, ref in zip(kept, refs) if keep and ref() is not None))+ kept.append(matched)+ refs.append(weakref.ref(metrics))+ yield metrics++ monkeypatch.setattr(storage, "iter_all", tracked_iter_all)++ page = storage.query(model="claude", limit=limit, offset=offset)++ expected = sorted(matches, key=lambda m: (m.timestamp, m.request_id), reverse=True)+ assert [m.request_id for m in page] == [m.request_id for m in expected[offset : offset + limit]]+ assert len(refs) == len(rows)+ # Selection may hold the requested window plus at most one matching record+ # still in flight while the next record is pulled; it must never retain the+ # whole match set.+ assert max(retained) <= window + 1+ assert max(retained) < len(matches)