Google Flights MCP, CLI and Python Library
chore: remove stale benchmark scripts (#183)
Punit Arani4495d55f28f5
diff --git a/scripts/benchmarks/__init__.py b/scripts/benchmarks/__init__.pydeleted file mode 100644index be015f6..0000000--- a/scripts/benchmarks/__init__.py+++ /dev/null@@ -1,7 +0,0 @@-"""Benchmark suite for ``fli`` parsing and concurrency.--Run ``python scripts/benchmarks/bench.py`` to print a side-by-side comparison-of the sequential and parallel code paths on the same hardware. All-benchmarks are deterministic — they replay captured fixtures and mock the-HTTP layer with a controlled-latency stub.-"""diff --git a/scripts/benchmarks/_fixtures.py b/scripts/benchmarks/_fixtures.pydeleted file mode 100644index 9560410..0000000--- a/scripts/benchmarks/_fixtures.py+++ /dev/null@@ -1,169 +0,0 @@-"""Synthetic fixture builders for the benchmark suite.--Captured fixtures only give us one shape per file. To cover scaling-behaviour (many rows, deep layovers, etc.) we synthesise additional-fixtures from primitives that mirror the live response layout. The-synthesisers live here so both the bench and the comparison scripts-share one source of truth.-"""--from __future__ import annotations--import json-from datetime import datetime, timedelta---def _leg(- *,- dep="JFK",- arr="LAX",- dep_date=(2026, 7, 15),- arr_date=(2026, 7, 15),- dep_time=(8, 0),- arr_time=(11, 30),- duration=210,- airline="DL",- flight_no="123",- aircraft="Boeing 737",-):- """Mirror the 33-element leg array layout from Google's response."""- leg = [None] * 33- leg[3] = dep- leg[4] = f"{dep} Airport"- leg[5] = f"{arr} Airport"- leg[6] = arr- leg[8] = list(dep_time)- leg[10] = list(arr_time)- leg[11] = duration- leg[12] = [None, True, None, None, None, True, None, None, None, True, None, 2]- leg[14] = "31 in"- leg[17] = aircraft- leg[19] = False- leg[20] = list(dep_date)- leg[21] = list(arr_date)- leg[22] = [airline, flight_no, None, "Carrier Name"]- leg[25] = 1- leg[30] = "31 inches"- leg[31] = 224000- leg[32] = 2- return leg---def _row(legs: list, *, price: float = 200.0, currency_token: str = "CAUaAlVTKAEgASoCVVMyAlVT"):- """Mirror the 11-element row + 25-element detail layouts."""- detail = [None] * 25- detail[0] = legs[0][22][0]- detail[1] = [legs[0][22][3]]- detail[2] = legs- detail[3] = legs[0][3]- detail[4] = legs[0][20]- detail[5] = legs[0][8]- detail[6] = legs[-1][6]- detail[7] = legs[-1][21]- detail[8] = legs[-1][10]- detail[9] = sum(leg[11] for leg in legs)- detail[12] = False- emissions = [None] * 18- emissions[3] = -10- emissions[7] = 220000- emissions[8] = 245000- emissions[10] = 280000- emissions[11] = 1- detail[22] = emissions- row = [None] * 11- row[0] = detail- row[1] = [[None, price], currency_token]- row[8] = f"BOOKING_TOKEN_{int(price)}"- row[10] = False- return row---def _nonstop_row(i: int) -> list:- return _row([_leg(flight_no=str(100 + i))], price=200.0 + i)---def _multi_leg_row(i: int) -> list:- leg1 = _leg(- dep="JFK",- arr="CDG",- dep_time=(19, 0),- arr_time=(9, 0),- dep_date=(2026, 7, 15),- arr_date=(2026, 7, 16),- duration=480,- airline="AF",- flight_no=f"{300 + i}",- )- leg2 = _leg(- dep="CDG",- arr="ATH",- dep_time=(13, 0),- arr_time=(17, 0),- dep_date=(2026, 7, 16),- arr_date=(2026, 7, 16),- duration=240,- airline="AF",- flight_no=f"{800 + i}",- )- return _row([leg1, leg2], price=650.0 + i)---def synthetic_flight_response(*, rows: int, multi_leg: bool = False) -> str:- """Build a fixture body with ``rows`` flight rows in the live wire format."""- builder = _multi_leg_row if multi_leg else _nonstop_row- flight_rows = [builder(i) for i in range(rows)]- # Match the live layout: inner[2][0] is the flight-row list, inner[3][0] also.- inner = [None, None, [flight_rows], [[]]]- return ")]}'\n" + json.dumps([["wrb.fr", None, json.dumps(inner)]])---def synthetic_multi_chunk_flight_response(*, rows_per_chunk: int, chunks: int) -> str:- r"""Build a multi-chunk wire body to exercise the streaming reader.-- Google's framing: ``<len>\n<json_bytes>\n``. The length header counts- the chunk's bytes — JSON + trailing newline — plus 1 (the convention- the reader compensates for with ``length - 1``).- """- parts = [")]}'\n\n"]- for c in range(chunks):- rows = [_nonstop_row(c * rows_per_chunk + i) for i in range(rows_per_chunk)]- inner = [None, None, [rows], [[]]]- payload = json.dumps([["wrb.fr", None, json.dumps(inner)]])- encoded = payload.encode("utf-8")- chunk_bytes = len(encoded) + 1 # +1 for trailing newline counted in the chunk- parts.append(f"{chunk_bytes + 1}\n")- parts.append(payload)- parts.append("\n")- return "".join(parts)---def synthetic_date_response(days: int = 61) -> str:- """Build a ``GetCalendarGraph`` body with ``days`` date entries."""- base = datetime(2026, 7, 1)- entries = [- [- (base + timedelta(days=i)).strftime("%Y-%m-%d"),- None,- [[None, 150 + i], "CAUaAlVTKAEgASoCVVMyAlVT"],- ]- for i in range(days)- ]- inner = json.dumps([None, None, entries])- return ")]}'\n" + json.dumps([["wrb.fr", None, inner]])---def synthetic_booking_response(*, vendor_count: int) -> str:- """Build a ``GetBookingResults`` body with ``vendor_count`` options."""- booking_rows = []- for i in range(vendor_count):- row = [None] * 25- row[0] = i- row[1] = [[f"V{i:02d}", f"Vendor {i}", None, i % 2 == 0]]- row[3] = [["AA", str(100 + i)]]- row[5] = [f"https://vendor{i}.example/book", None, ["https://google.com/travel/clk?v=1"]]- row[7] = [[None, 250.0 + i], "CAUaAlVTKAEgASoCVVMyAlVT"]- row[21] = [None, None, None, f"Fare class {i}"]- booking_rows.append(row)-- inner = [None, [booking_rows]]- return ")]}'\n" + json.dumps([["wrb.fr", None, json.dumps(inner)]])diff --git a/scripts/benchmarks/_harness.py b/scripts/benchmarks/_harness.pydeleted file mode 100644index cd0631f..0000000--- a/scripts/benchmarks/_harness.py+++ /dev/null@@ -1,118 +0,0 @@-"""Shared timing / reporting helpers for the benchmark suite.--Everything here is deliberately framework-free: no pytest-benchmark, no-external dependencies beyond the standard library. The goal is a-reproducible apples-to-apples wall-clock and CPU-time comparison.-"""--from __future__ import annotations--import statistics-import time-from collections.abc import Callable-from dataclasses import dataclass, field-from typing import Any---@dataclass-class BenchResult:- """Statistical summary of a single benchmark scenario."""-- name: str- iterations: int- wall_ms: list[float] = field(default_factory=list)- cpu_ms: list[float] = field(default_factory=list)- payload: Any = None-- @property- def wall_mean(self) -> float:- return statistics.mean(self.wall_ms) if self.wall_ms else 0.0-- @property- def wall_p50(self) -> float:- return statistics.median(self.wall_ms) if self.wall_ms else 0.0-- @property- def wall_p95(self) -> float:- return _percentile(self.wall_ms, 95)-- @property- def wall_stdev(self) -> float:- return statistics.stdev(self.wall_ms) if len(self.wall_ms) >= 2 else 0.0-- @property- def cpu_mean(self) -> float:- return statistics.mean(self.cpu_ms) if self.cpu_ms else 0.0-- def as_row(self) -> tuple[str, str, str, str, str, str]:- """Return a tuple ready to drop into a printed table."""- return (- self.name,- f"{self.iterations}",- f"{self.wall_mean:7.2f}",- f"{self.wall_p50:7.2f}",- f"{self.wall_p95:7.2f}",- f"{self.cpu_mean:7.2f}",- )---def _percentile(values: list[float], pct: float) -> float:- if not values:- return 0.0- if len(values) == 1:- return values[0]- ordered = sorted(values)- k = (len(ordered) - 1) * (pct / 100.0)- f = int(k)- c = min(f + 1, len(ordered) - 1)- if f == c:- return ordered[f]- return ordered[f] + (ordered[c] - ordered[f]) * (k - f)---def time_callable(- fn: Callable[[], Any],- *,- iterations: int,- warmup: int = 1,- name: str | None = None,-) -> BenchResult:- """Run ``fn`` ``iterations`` times after ``warmup`` discard runs.-- Returns wall-clock and process CPU time per iteration. The payload of- the last run is retained so callers can sanity-check the output.- """- for _ in range(warmup):- fn()- result = BenchResult(name=name or fn.__name__, iterations=iterations)- payload = None- for _ in range(iterations):- t_wall = time.perf_counter()- t_cpu = time.process_time()- payload = fn()- result.wall_ms.append((time.perf_counter() - t_wall) * 1000.0)- result.cpu_ms.append((time.process_time() - t_cpu) * 1000.0)- result.payload = payload- return result---def print_table(title: str, results: list[BenchResult]) -> None:- """Pretty-print a comparison table grouped under ``title``."""- header = ("scenario", "iter", "mean ms", "p50 ms", "p95 ms", "cpu ms")- rows = [r.as_row() for r in results]- widths = [max(len(header[i]), *(len(row[i]) for row in rows)) for i in range(len(header))]- print()- print(title)- print("-" * len(title))- line = " ".join(h.ljust(widths[i]) for i, h in enumerate(header))- print(line)- print(" ".join("-" * w for w in widths))- for row in rows:- print(" ".join(c.ljust(widths[i]) for i, c in enumerate(row)))---def speedup(baseline: BenchResult, candidate: BenchResult) -> float:- """Return baseline / candidate wall-clock mean (>1 means candidate is faster)."""- if candidate.wall_mean <= 0:- return float("inf")- return baseline.wall_mean / candidate.wall_meandiff --git a/scripts/benchmarks/_mocks.py b/scripts/benchmarks/_mocks.pydeleted file mode 100644index b6baed3..0000000--- a/scripts/benchmarks/_mocks.py+++ /dev/null@@ -1,86 +0,0 @@-"""HTTP mock infrastructure for deterministic concurrency benchmarks.--The fli library talks to Google Flights through a single ``Client`` wrapper-(`fli.search.client.Client`). We swap that out with a controlled-latency-stub so we can measure how well the search code parallelises *its* work,-without actually touching the network or Google's 10 req/sec ceiling.--Each stub method:--* Sleeps for a configurable number of milliseconds (simulated network RTT).-* Returns a pre-baked ``Response``-shaped object whose ``.text`` is one of- the captured fixtures.-* Increments a thread-safe counter so the benchmark can verify the search- layer actually issued the expected number of requests.-"""--from __future__ import annotations--import threading-import time-from pathlib import Path-from typing import Any--FIXTURE_DIR = Path(__file__).resolve().parents[2] / "tests" / "search" / "fixtures"---class _FakeResponse:- """Minimal stand-in for ``curl_cffi.requests.Response``."""-- __slots__ = ("text", "status_code")-- def __init__(self, text: str, status_code: int = 200):- self.text = text- self.status_code = status_code-- def raise_for_status(self) -> None:- if self.status_code >= 400:- raise RuntimeError(f"HTTP {self.status_code}")---class FakeClient:- """Thread-safe controlled-latency HTTP stub.-- ``post()`` / ``get()`` block for ``latency_ms`` milliseconds and return- the configured fixture body. The same fixture is returned on every- call — search logic uses the response shape, not its content, for the- parts we want to measure.- """-- def __init__(- self,- fixture_text: str,- *,- latency_ms: float = 100.0,- ):- self._fixture = fixture_text- self._latency_s = latency_ms / 1000.0- self._lock = threading.Lock()- self.calls = 0- self.concurrent_high_water = 0- self._in_flight = 0-- def _sleep(self) -> None:- with self._lock:- self.calls += 1- self._in_flight += 1- if self._in_flight > self.concurrent_high_water:- self.concurrent_high_water = self._in_flight- try:- time.sleep(self._latency_s)- finally:- with self._lock:- self._in_flight -= 1-- def post(self, url: str, **kwargs: Any) -> _FakeResponse:- self._sleep()- return _FakeResponse(self._fixture)-- def get(self, url: str, **kwargs: Any) -> _FakeResponse:- self._sleep()- return _FakeResponse(self._fixture)---def load_fixture(name: str) -> str:- """Read a captured response body from ``tests/search/fixtures/``."""- return (FIXTURE_DIR / name).read_text()diff --git a/scripts/benchmarks/bench.py b/scripts/benchmarks/bench.pydeleted file mode 100644index e1935e2..0000000--- a/scripts/benchmarks/bench.py+++ /dev/null@@ -1,498 +0,0 @@-"""Comprehensive ``fli`` parsing & concurrency benchmark suite (30+ scenarios).--Each scenario is deterministic — it replays a real or synthetic fixture-through the same parser code production uses, with HTTP swapped for a-controlled-latency stub. The output is grouped into five sections so-regressions are easy to spot:--1. **Parsing (CPU only)** — scales rows from 1 → 1000 to expose the- per-row constant factor and any super-linear costs.-2. **Wire-format reading** — single vs multi-chunk responses, the- currency-token cache, booking decode at multiple vendor counts.-3. **End-to-end search (I/O bound, mocked HTTP)** — one-way at three- latencies, round-trip across ``top_n`` ∈ {2, 5, 10}, multi-city.-4. **Date-range chunking** — 30 / 90 / 180 / 305-day ranges to verify- parallelism scales with chunk count.-5. **Concurrency primitives** — bare-metal numbers for- ``parallel_map`` and ``TokenBucketRateLimiter``.--Usage:-- uv run python scripts/benchmarks/bench.py [--iterations N] [--latency-ms M]-"""--from __future__ import annotations--import argparse-import sys-import threading-import time-from copy import deepcopy-from datetime import datetime, timedelta-from pathlib import Path--ROOT = Path(__file__).resolve().parents[2]-sys.path.insert(0, str(ROOT))--from fli.models import ( # noqa: E402- Airport,- DateSearchFilters,- FlightSearchFilters,- FlightSegment,- PassengerInfo,- TripType,-)-from fli.search import SearchDates, SearchFlights # noqa: E402-from fli.search._concurrency import ( # noqa: E402- TokenBucketRateLimiter,- parallel_map,-)-from fli.search._decoders import parse_booking_chunk, parse_flight_row # noqa: E402-from fli.search._wire import iter_wrb_chunks, parse_first_wrb_payload # noqa: E402-from scripts.benchmarks._fixtures import ( # noqa: E402- synthetic_booking_response,- synthetic_date_response,- synthetic_flight_response,- synthetic_multi_chunk_flight_response,-)-from scripts.benchmarks._harness import ( # noqa: E402- BenchResult,- print_table,- time_callable,-)-from scripts.benchmarks._mocks import FakeClient, load_fixture # noqa: E402--# ----------------------------------------------------------------------------# Filter builders-# ------------------------------------------------------------------------------def _today_plus(days: int) -> str:- return (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d")---def _one_way_filters() -> FlightSearchFilters:- return FlightSearchFilters(- trip_type=TripType.ONE_WAY,- passenger_info=PassengerInfo(adults=1),- flight_segments=[- FlightSegment(- departure_airport=[[Airport.JFK, 0]],- arrival_airport=[[Airport.LAX, 0]],- travel_date=_today_plus(30),- )- ],- )---def _round_trip_filters() -> FlightSearchFilters:- return FlightSearchFilters(- trip_type=TripType.ROUND_TRIP,- passenger_info=PassengerInfo(adults=1),- flight_segments=[- FlightSegment(- departure_airport=[[Airport.JFK, 0]],- arrival_airport=[[Airport.LAX, 0]],- travel_date=_today_plus(30),- ),- FlightSegment(- departure_airport=[[Airport.LAX, 0]],- arrival_airport=[[Airport.JFK, 0]],- travel_date=_today_plus(37),- ),- ],- )---def _multi_city_filters(n_segments: int) -> FlightSearchFilters:- cities = [Airport.JFK, Airport.LAX, Airport.SFO, Airport.ORD, Airport.MIA, Airport.BOS]- segments = []- for i in range(n_segments):- dep = cities[i % len(cities)]- arr = cities[(i + 1) % len(cities)]- segments.append(- FlightSegment(- departure_airport=[[dep, 0]],- arrival_airport=[[arr, 0]],- travel_date=_today_plus(30 + i * 5),- )- )- return FlightSearchFilters(- trip_type=TripType.MULTI_CITY,- passenger_info=PassengerInfo(adults=1),- flight_segments=segments,- )---def _date_filters(days: int) -> DateSearchFilters:- start = _today_plus(7)- end = (datetime.strptime(start, "%Y-%m-%d") + timedelta(days=days - 1)).strftime("%Y-%m-%d")- return DateSearchFilters(- trip_type=TripType.ONE_WAY,- passenger_info=PassengerInfo(adults=1),- flight_segments=[- FlightSegment(- departure_airport=[[Airport.JFK, 0]],- arrival_airport=[[Airport.LAX, 0]],- travel_date=start,- )- ],- from_date=start,- to_date=end,- )---# ----------------------------------------------------------------------------# Helpers-# ------------------------------------------------------------------------------def _rows_from_fixture(fixture: str) -> list:- inner = parse_first_wrb_payload(fixture)- if inner is None:- return []- return [item for i in (2, 3) if isinstance(inner[i], list) for item in inner[i][0]]---def _wrap(meta: dict, fake: FakeClient) -> dict:- meta["calls"] = fake.calls- meta["concurrent_max"] = fake.concurrent_high_water- return meta---# ----------------------------------------------------------------------------# Section 1 — Parsing scaling (CPU only)-# ------------------------------------------------------------------------------def parse_rows_scenario(name: str, rows: list, iterations: int) -> BenchResult:- def run():- return [parse_flight_row(r) for r in rows]-- res = time_callable(run, iterations=iterations, name=name)- res.payload = {"rows": len(rows)}- return res---def bench_parsing_section(iters: int) -> list[BenchResult]:- """Cover 1, 10, 50, 200, 1000 row volumes + multi-leg + diverse currencies."""- results: list[BenchResult] = []-- # Real fixture- real_body = load_fixture("flight_search_jfk_lax_oneway_usd.bin")- real_rows = _rows_from_fixture(real_body)- results.append(parse_rows_scenario("parse 1 row", real_rows[:1], iters * 40))- results.append(parse_rows_scenario("parse 10 rows", real_rows[:10], iters * 20))- results.append(parse_rows_scenario("parse 28 rows (real)", real_rows, iters * 10))-- # Synthetic- syn_50 = _rows_from_fixture(synthetic_flight_response(rows=50))- syn_200 = _rows_from_fixture(synthetic_flight_response(rows=200))- syn_1000 = _rows_from_fixture(synthetic_flight_response(rows=1000))- results.append(parse_rows_scenario("parse 50 rows", syn_50, iters * 4))- results.append(parse_rows_scenario("parse 200 rows", syn_200, iters * 2))- results.append(parse_rows_scenario("parse 1000 rows", syn_1000, iters))-- # Multi-leg (layover derivation)- syn_ml = _rows_from_fixture(synthetic_flight_response(rows=50, multi_leg=True))- results.append(parse_rows_scenario("parse 50 rows (multi-leg)", syn_ml, iters * 4))-- return results---# ----------------------------------------------------------------------------# Section 2 — Wire-format reading-# ------------------------------------------------------------------------------def bench_wire_section(iters: int) -> list[BenchResult]:- """Wire decoder timings — single / multi chunk and booking parsing."""- results: list[BenchResult] = []-- real = load_fixture("flight_search_jfk_lax_oneway_usd.bin")- one_chunk = synthetic_multi_chunk_flight_response(rows_per_chunk=50, chunks=1)- two_chunk = synthetic_multi_chunk_flight_response(rows_per_chunk=50, chunks=2)- five_chunk = synthetic_multi_chunk_flight_response(rows_per_chunk=20, chunks=5)-- results.append(- time_callable(- lambda: parse_first_wrb_payload(real),- iterations=iters * 20,- name="iter_wrb: real 28-row body",- )- )- results.append(- time_callable(- lambda: list(iter_wrb_chunks(one_chunk)),- iterations=iters * 20,- name="iter_wrb: synthetic 1 chunk",- )- )- results.append(- time_callable(- lambda: list(iter_wrb_chunks(two_chunk)),- iterations=iters * 10,- name="iter_wrb: synthetic 2 chunks",- )- )- results.append(- time_callable(- lambda: list(iter_wrb_chunks(five_chunk)),- iterations=iters * 10,- name="iter_wrb: synthetic 5 chunks",- )- )-- booking_small = synthetic_booking_response(vendor_count=5)- booking_large = synthetic_booking_response(vendor_count=50)- results.append(- time_callable(- lambda: parse_booking_chunk(parse_first_wrb_payload(booking_small)),- iterations=iters * 10,- name="parse_booking: 5 vendors",- )- )- results.append(- time_callable(- lambda: parse_booking_chunk(parse_first_wrb_payload(booking_large)),- iterations=iters * 5,- name="parse_booking: 50 vendors",- )- )-- return results---# ----------------------------------------------------------------------------# Section 3 — End-to-end search (mocked HTTP, varying latency)-# ------------------------------------------------------------------------------def bench_search_section(iters: int) -> list[BenchResult]:- """End-to-end search wall clock with mocked HTTP at varying latency."""- fixture = load_fixture("flight_search_jfk_lax_oneway_usd.bin")- results: list[BenchResult] = []-- def make_one_way(latency_ms: float):- fake = FakeClient(fixture, latency_ms=latency_ms)-- def run():- search = SearchFlights()- search.client = fake- return search.search(_one_way_filters())-- res = time_callable(run, iterations=iters, name=f"one-way @ {int(latency_ms)}ms")- res.payload = _wrap({}, fake)- return res-- def make_round_trip(top_n: int, latency_ms: float = 100.0):- fake = FakeClient(fixture, latency_ms=latency_ms)-- def run():- search = SearchFlights()- search.client = fake- return search.search(_round_trip_filters(), top_n=top_n)-- res = time_callable(- run,- iterations=iters,- name=f"round-trip top_n={top_n} @ {int(latency_ms)}ms",- )- res.payload = _wrap({}, fake)- return res-- def make_multi_city(n_segments: int, latency_ms: float = 100.0):- fake = FakeClient(fixture, latency_ms=latency_ms)-- def run():- search = SearchFlights()- search.client = fake- f = _multi_city_filters(n_segments)- return search.search(f, top_n=3)-- res = time_callable(- run,- iterations=iters,- name=f"multi-city {n_segments}-seg @ {int(latency_ms)}ms",- )- res.payload = _wrap({}, fake)- return res-- results.append(make_one_way(50.0))- results.append(make_one_way(120.0))- results.append(make_one_way(500.0))- results.append(make_round_trip(top_n=2))- results.append(make_round_trip(top_n=5))- results.append(make_round_trip(top_n=10))- results.append(make_multi_city(n_segments=3))-- return results---# ----------------------------------------------------------------------------# Section 4 — Date-range chunking-# ------------------------------------------------------------------------------def bench_dates_section(iters: int) -> list[BenchResult]:- """Date-range searches — chunk count grows linearly with range size."""- date_body = synthetic_date_response(days=61)- results: list[BenchResult] = []-- def make(days: int, latency_ms: float = 120.0):- fake = FakeClient(date_body, latency_ms=latency_ms)-- def run():- search = SearchDates()- search.client = fake- return search.search(deepcopy(_date_filters(days)))-- chunks = (days + 60) // 61- res = time_callable(- run,- iterations=iters,- name=f"dates {days}d ({chunks} chunks)",- )- res.payload = _wrap({}, fake)- return res-- results.append(make(30)) # 1 chunk- results.append(make(90)) # 2 chunks- results.append(make(180)) # 3 chunks- results.append(make(244)) # 4 chunks- results.append(make(305)) # 5 chunks (max range)-- return results---# ----------------------------------------------------------------------------# Section 5 — Concurrency primitives-# ------------------------------------------------------------------------------def bench_concurrency_section(iters: int) -> list[BenchResult]:- """Bare-metal numbers for the primitives the search code depends on."""- results: list[BenchResult] = []-- # parallel_map: small list (5 items, 10ms each)- def pmap_small():- return parallel_map(lambda x: time.sleep(0.01) or x, list(range(5)), max_workers=5)-- results.append(time_callable(pmap_small, iterations=iters, name="parallel_map: 5x10ms"))-- # parallel_map: large list (20 items, 5ms each)- def pmap_large():- return parallel_map(lambda x: time.sleep(0.005) or x, list(range(20)), max_workers=10)-- results.append(time_callable(pmap_large, iterations=iters, name="parallel_map: 20x5ms"))-- # parallel_map: synchronous fast path- def pmap_sync():- return parallel_map(lambda x: x * 2, [42])-- results.append(- time_callable(pmap_sync, iterations=iters * 100, name="parallel_map: 1 item (sync)")- )-- # parallel_map: CPU work — token bucket overhead shouldn't matter- def pmap_cpu():- return parallel_map(lambda x: sum(range(x)), [10000] * 8, max_workers=4)-- results.append(time_callable(pmap_cpu, iterations=iters, name="parallel_map: 8 CPU jobs"))-- # TokenBucket: uncontended acquire- bucket_uncon = TokenBucketRateLimiter(calls=1000, period=1.0)- results.append(- time_callable(- lambda: bucket_uncon.acquire(),- iterations=iters * 200,- name="bucket acquire (uncontended)",- )- )-- # TokenBucket: contended (5 threads sharing 100/sec budget)- bucket_con = TokenBucketRateLimiter(calls=100, period=1.0)-- def contended():- threads = []- for _ in range(5):- t = threading.Thread(target=lambda: [bucket_con.acquire() for _ in range(4)])- threads.append(t)- t.start()- for t in threads:- t.join()-- results.append(- time_callable(contended, iterations=iters, name="bucket: 5 threads x 4 acquires")- )-- return results---# ----------------------------------------------------------------------------# Entry point-# ------------------------------------------------------------------------------def main() -> int:- """Run all five sections and print a single combined report."""- parser = argparse.ArgumentParser(description=__doc__)- parser.add_argument("--iterations", type=int, default=5)- parser.add_argument(- "--latency-ms",- type=float,- default=120.0,- help="Used wherever a benchmark doesn't fix its own latency.",- )- args = parser.parse_args()- iters = args.iterations- print(f"Running benchmark suite: {iters} iterations per scenario.\n")-- sections = [- ("Section 1 — Parsing (CPU)", bench_parsing_section(iters)),- ("Section 2 — Wire format", bench_wire_section(iters)),- ("Section 3 — End-to-end search (mocked HTTP)", bench_search_section(iters)),- ("Section 4 — Date-range chunking", bench_dates_section(iters)),- ("Section 5 — Concurrency primitives", bench_concurrency_section(iters)),- ]-- total = sum(len(r) for _, r in sections)- print(f"Total scenarios: {total}\n")- for title, results in sections:- print_table(title, results)-- # Concurrency observations for the search/date sections.- print("\nObserved request concurrency:")- for title, results in sections:- if "Section 3" in title or "Section 4" in title:- for r in results:- meta = r.payload or {}- calls = meta.get("calls", 0) / max(r.iterations, 1)- peak = meta.get("concurrent_max", 1)- print(f" {r.name:40s} calls/run={calls:5.1f} peak in-flight={peak}")-- # Top-level throughput summary.- print("\nKey throughput metrics:")- for title, results in sections:- if "Section 1" in title:- for r in results:- rows = (r.payload or {}).get("rows", 0)- if rows and r.wall_mean > 0:- throughput = rows * 1000.0 / r.wall_mean- print(f" {r.name:30s} {throughput:>12,.0f} rows/sec")-- return 0---if __name__ == "__main__":- raise SystemExit(main())---# Re-exports for the comparison driver (compare.py).-__all__ = [- "bench_concurrency_section",- "bench_dates_section",- "bench_parsing_section",- "bench_search_section",- "bench_wire_section",-]diff --git a/scripts/benchmarks/compare.py b/scripts/benchmarks/compare.pydeleted file mode 100644index 0ee79c9..0000000--- a/scripts/benchmarks/compare.py+++ /dev/null@@ -1,130 +0,0 @@-"""Sequential-vs-parallel comparison driver across every scenario in ``bench.py``.--Runs each scenario twice — once with the shared executor forced down to a-single worker (so ``parallel_map`` falls back to its synchronous loop),-once with the default capacity — and prints a side-by-side table with-speedup factors.--Scenarios where parallelism makes no difference (pure CPU parsing,-single-chunk wire reads) still appear so we can confirm that the-parallel mode never *regresses* the fast path.--Usage:-- uv run python scripts/benchmarks/compare.py [--iterations N]-"""--from __future__ import annotations--import argparse-import sys-from pathlib import Path--ROOT = Path(__file__).resolve().parents[2]-sys.path.insert(0, str(ROOT))--# Force-toggle the shared executor before importing any scenario builders-# so they all see the configured worker cap.-import fli.search._concurrency as _conc # noqa: E402-from scripts.benchmarks._harness import BenchResult, speedup # noqa: E402-from scripts.benchmarks.bench import ( # noqa: E402- bench_concurrency_section,- bench_dates_section,- bench_parsing_section,- bench_search_section,- bench_wire_section,-)---def _set_parallel(enabled: bool) -> None:- _conc.shutdown_executor()- _conc.configure_concurrency(10 if enabled else 1)---def _collect(iters: int) -> dict[str, list[BenchResult]]:- return {- "Parsing (CPU)": bench_parsing_section(iters),- "Wire format": bench_wire_section(iters),- "End-to-end search": bench_search_section(iters),- "Date-range chunking": bench_dates_section(iters),- "Concurrency primitives": bench_concurrency_section(iters),- }---def _print_section(title: str, seq: list[BenchResult], par: list[BenchResult]) -> None:- print(f"\n{title}")- print("-" * len(title))- header = ("scenario", "seq ms", "par ms", "speedup", "saved ms")- rows: list[tuple[str, ...]] = []- for s, p in zip(seq, par, strict=False):- sup = speedup(s, p)- saved = s.wall_mean - p.wall_mean- rows.append(- (- s.name,- f"{s.wall_mean:8.2f}",- f"{p.wall_mean:8.2f}",- f"{sup:5.2f}x",- f"{saved:+8.2f}",- )- )- widths = [max(len(header[i]), *(len(r[i]) for r in rows)) for i in range(len(header))]- print(" ".join(h.ljust(widths[i]) for i, h in enumerate(header)))- print(" ".join("-" * w for w in widths))- for row in rows:- print(" ".join(c.ljust(widths[i]) for i, c in enumerate(row)))---def _print_top_speedups(all_seq: list[BenchResult], all_par: list[BenchResult]) -> None:- pairs = [(s, p, speedup(s, p)) for s, p in zip(all_seq, all_par, strict=False)]- pairs.sort(key=lambda x: x[2], reverse=True)- print("\nTop speedups overall:")- print("-" * 22)- for s, p, sup in pairs[:8]:- print(- f" {s.name:45s} {s.wall_mean:7.2f}ms → {p.wall_mean:7.2f}ms "- f"({sup:5.2f}x, saved {s.wall_mean - p.wall_mean:+7.2f}ms)"- )---def main() -> int:- """Run every scenario in sequential mode, then parallel, then diff."""- parser = argparse.ArgumentParser(description=__doc__)- parser.add_argument("--iterations", type=int, default=3)- args = parser.parse_args()-- iters = args.iterations- print(f"Comparison run: {iters} iterations per scenario, parallel toggled via executor cap.\n")-- _set_parallel(False)- print("[1/2] Running SEQUENTIAL (max_workers=1)...")- seq_sections = _collect(iters)-- _set_parallel(True)- print("[2/2] Running PARALLEL (max_workers=10)...")- par_sections = _collect(iters)-- flat_seq: list[BenchResult] = []- flat_par: list[BenchResult] = []- for title, seq in seq_sections.items():- par = par_sections[title]- _print_section(title, seq, par)- flat_seq.extend(seq)- flat_par.extend(par)-- _print_top_speedups(flat_seq, flat_par)-- # Quick interpretive summary.- total_seq = sum(s.wall_mean for s in flat_seq)- total_par = sum(p.wall_mean for p in flat_par)- print(- f"\nAggregate across {len(flat_seq)} scenarios: "- f"sequential={total_seq:.0f}ms parallel={total_par:.0f}ms "- f"saved={total_seq - total_par:.0f}ms ({total_seq / max(total_par, 1e-6):.2f}x)"- )-- return 0---if __name__ == "__main__":- raise SystemExit(main())