Skip to content

armgit / fli

Public Imported from GitHub
0

Google Flights MCP, CLI and Python Library

chore: remove stale benchmark scripts (#183)

Punit Arani4495d55f28f5
diff --git a/scripts/benchmarks/__init__.py b/scripts/benchmarks/__init__.py
deleted file mode 100644
index be015f6..0000000
--- a/scripts/benchmarks/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-"""Benchmark suite for ``fli`` parsing and concurrency.
-
-Run ``python scripts/benchmarks/bench.py`` to print a side-by-side comparison
-of the sequential and parallel code paths on the same hardware. All
-benchmarks are deterministic — they replay captured fixtures and mock the
-HTTP layer with a controlled-latency stub.
-"""
diff --git a/scripts/benchmarks/_fixtures.py b/scripts/benchmarks/_fixtures.py
deleted file mode 100644
index 9560410..0000000
--- a/scripts/benchmarks/_fixtures.py
+++ /dev/null
@@ -1,169 +0,0 @@
-"""Synthetic fixture builders for the benchmark suite.
-
-Captured fixtures only give us one shape per file. To cover scaling
-behaviour (many rows, deep layovers, etc.) we synthesise additional
-fixtures from primitives that mirror the live response layout. The
-synthesisers live here so both the bench and the comparison scripts
-share one source of truth.
-"""
-
-from __future__ import annotations
-
-import json
-from datetime import datetime, timedelta
-
-
-def _leg(
- *,
- dep="JFK",
- arr="LAX",
- dep_date=(2026, 7, 15),
- arr_date=(2026, 7, 15),
- dep_time=(8, 0),
- arr_time=(11, 30),
- duration=210,
- airline="DL",
- flight_no="123",
- aircraft="Boeing 737",
-):
- """Mirror the 33-element leg array layout from Google's response."""
- leg = [None] * 33
- leg[3] = dep
- leg[4] = f"{dep} Airport"
- leg[5] = f"{arr} Airport"
- leg[6] = arr
- leg[8] = list(dep_time)
- leg[10] = list(arr_time)
- leg[11] = duration
- leg[12] = [None, True, None, None, None, True, None, None, None, True, None, 2]
- leg[14] = "31 in"
- leg[17] = aircraft
- leg[19] = False
- leg[20] = list(dep_date)
- leg[21] = list(arr_date)
- leg[22] = [airline, flight_no, None, "Carrier Name"]
- leg[25] = 1
- leg[30] = "31 inches"
- leg[31] = 224000
- leg[32] = 2
- return leg
-
-
-def _row(legs: list, *, price: float = 200.0, currency_token: str = "CAUaAlVTKAEgASoCVVMyAlVT"):
- """Mirror the 11-element row + 25-element detail layouts."""
- detail = [None] * 25
- detail[0] = legs[0][22][0]
- detail[1] = [legs[0][22][3]]
- detail[2] = legs
- detail[3] = legs[0][3]
- detail[4] = legs[0][20]
- detail[5] = legs[0][8]
- detail[6] = legs[-1][6]
- detail[7] = legs[-1][21]
- detail[8] = legs[-1][10]
- detail[9] = sum(leg[11] for leg in legs)
- detail[12] = False
- emissions = [None] * 18
- emissions[3] = -10
- emissions[7] = 220000
- emissions[8] = 245000
- emissions[10] = 280000
- emissions[11] = 1
- detail[22] = emissions
- row = [None] * 11
- row[0] = detail
- row[1] = [[None, price], currency_token]
- row[8] = f"BOOKING_TOKEN_{int(price)}"
- row[10] = False
- return row
-
-
-def _nonstop_row(i: int) -> list:
- return _row([_leg(flight_no=str(100 + i))], price=200.0 + i)
-
-
-def _multi_leg_row(i: int) -> list:
- leg1 = _leg(
- dep="JFK",
- arr="CDG",
- dep_time=(19, 0),
- arr_time=(9, 0),
- dep_date=(2026, 7, 15),
- arr_date=(2026, 7, 16),
- duration=480,
- airline="AF",
- flight_no=f"{300 + i}",
- )
- leg2 = _leg(
- dep="CDG",
- arr="ATH",
- dep_time=(13, 0),
- arr_time=(17, 0),
- dep_date=(2026, 7, 16),
- arr_date=(2026, 7, 16),
- duration=240,
- airline="AF",
- flight_no=f"{800 + i}",
- )
- return _row([leg1, leg2], price=650.0 + i)
-
-
-def synthetic_flight_response(*, rows: int, multi_leg: bool = False) -> str:
- """Build a fixture body with ``rows`` flight rows in the live wire format."""
- builder = _multi_leg_row if multi_leg else _nonstop_row
- flight_rows = [builder(i) for i in range(rows)]
- # Match the live layout: inner[2][0] is the flight-row list, inner[3][0] also.
- inner = [None, None, [flight_rows], [[]]]
- return ")]}'\n" + json.dumps([["wrb.fr", None, json.dumps(inner)]])
-
-
-def synthetic_multi_chunk_flight_response(*, rows_per_chunk: int, chunks: int) -> str:
- r"""Build a multi-chunk wire body to exercise the streaming reader.
-
- Google's framing: ``<len>\n<json_bytes>\n``. The length header counts
- the chunk's bytes — JSON + trailing newline — plus 1 (the convention
- the reader compensates for with ``length - 1``).
- """
- parts = [")]}'\n\n"]
- for c in range(chunks):
- rows = [_nonstop_row(c * rows_per_chunk + i) for i in range(rows_per_chunk)]
- inner = [None, None, [rows], [[]]]
- payload = json.dumps([["wrb.fr", None, json.dumps(inner)]])
- encoded = payload.encode("utf-8")
- chunk_bytes = len(encoded) + 1 # +1 for trailing newline counted in the chunk
- parts.append(f"{chunk_bytes + 1}\n")
- parts.append(payload)
- parts.append("\n")
- return "".join(parts)
-
-
-def synthetic_date_response(days: int = 61) -> str:
- """Build a ``GetCalendarGraph`` body with ``days`` date entries."""
- base = datetime(2026, 7, 1)
- entries = [
- [
- (base + timedelta(days=i)).strftime("%Y-%m-%d"),
- None,
- [[None, 150 + i], "CAUaAlVTKAEgASoCVVMyAlVT"],
- ]
- for i in range(days)
- ]
- inner = json.dumps([None, None, entries])
- return ")]}'\n" + json.dumps([["wrb.fr", None, inner]])
-
-
-def synthetic_booking_response(*, vendor_count: int) -> str:
- """Build a ``GetBookingResults`` body with ``vendor_count`` options."""
- booking_rows = []
- for i in range(vendor_count):
- row = [None] * 25
- row[0] = i
- row[1] = [[f"V{i:02d}", f"Vendor {i}", None, i % 2 == 0]]
- row[3] = [["AA", str(100 + i)]]
- row[5] = [f"https://vendor{i}.example/book", None, ["https://google.com/travel/clk?v=1"]]
- row[7] = [[None, 250.0 + i], "CAUaAlVTKAEgASoCVVMyAlVT"]
- row[21] = [None, None, None, f"Fare class {i}"]
- booking_rows.append(row)
-
- inner = [None, [booking_rows]]
- return ")]}'\n" + json.dumps([["wrb.fr", None, json.dumps(inner)]])
diff --git a/scripts/benchmarks/_harness.py b/scripts/benchmarks/_harness.py
deleted file mode 100644
index cd0631f..0000000
--- a/scripts/benchmarks/_harness.py
+++ /dev/null
@@ -1,118 +0,0 @@
-"""Shared timing / reporting helpers for the benchmark suite.
-
-Everything here is deliberately framework-free: no pytest-benchmark, no
-external dependencies beyond the standard library. The goal is a
-reproducible apples-to-apples wall-clock and CPU-time comparison.
-"""
-
-from __future__ import annotations
-
-import statistics
-import time
-from collections.abc import Callable
-from dataclasses import dataclass, field
-from typing import Any
-
-
-@dataclass
-class BenchResult:
- """Statistical summary of a single benchmark scenario."""
-
- name: str
- iterations: int
- wall_ms: list[float] = field(default_factory=list)
- cpu_ms: list[float] = field(default_factory=list)
- payload: Any = None
-
- @property
- def wall_mean(self) -> float:
- return statistics.mean(self.wall_ms) if self.wall_ms else 0.0
-
- @property
- def wall_p50(self) -> float:
- return statistics.median(self.wall_ms) if self.wall_ms else 0.0
-
- @property
- def wall_p95(self) -> float:
- return _percentile(self.wall_ms, 95)
-
- @property
- def wall_stdev(self) -> float:
- return statistics.stdev(self.wall_ms) if len(self.wall_ms) >= 2 else 0.0
-
- @property
- def cpu_mean(self) -> float:
- return statistics.mean(self.cpu_ms) if self.cpu_ms else 0.0
-
- def as_row(self) -> tuple[str, str, str, str, str, str]:
- """Return a tuple ready to drop into a printed table."""
- return (
- self.name,
- f"{self.iterations}",
- f"{self.wall_mean:7.2f}",
- f"{self.wall_p50:7.2f}",
- f"{self.wall_p95:7.2f}",
- f"{self.cpu_mean:7.2f}",
- )
-
-
-def _percentile(values: list[float], pct: float) -> float:
- if not values:
- return 0.0
- if len(values) == 1:
- return values[0]
- ordered = sorted(values)
- k = (len(ordered) - 1) * (pct / 100.0)
- f = int(k)
- c = min(f + 1, len(ordered) - 1)
- if f == c:
- return ordered[f]
- return ordered[f] + (ordered[c] - ordered[f]) * (k - f)
-
-
-def time_callable(
- fn: Callable[[], Any],
- *,
- iterations: int,
- warmup: int = 1,
- name: str | None = None,
-) -> BenchResult:
- """Run ``fn`` ``iterations`` times after ``warmup`` discard runs.
-
- Returns wall-clock and process CPU time per iteration. The payload of
- the last run is retained so callers can sanity-check the output.
- """
- for _ in range(warmup):
- fn()
- result = BenchResult(name=name or fn.__name__, iterations=iterations)
- payload = None
- for _ in range(iterations):
- t_wall = time.perf_counter()
- t_cpu = time.process_time()
- payload = fn()
- result.wall_ms.append((time.perf_counter() - t_wall) * 1000.0)
- result.cpu_ms.append((time.process_time() - t_cpu) * 1000.0)
- result.payload = payload
- return result
-
-
-def print_table(title: str, results: list[BenchResult]) -> None:
- """Pretty-print a comparison table grouped under ``title``."""
- header = ("scenario", "iter", "mean ms", "p50 ms", "p95 ms", "cpu ms")
- rows = [r.as_row() for r in results]
- widths = [max(len(header[i]), *(len(row[i]) for row in rows)) for i in range(len(header))]
- print()
- print(title)
- print("-" * len(title))
- line = " ".join(h.ljust(widths[i]) for i, h in enumerate(header))
- print(line)
- print(" ".join("-" * w for w in widths))
- for row in rows:
- print(" ".join(c.ljust(widths[i]) for i, c in enumerate(row)))
-
-
-def speedup(baseline: BenchResult, candidate: BenchResult) -> float:
- """Return baseline / candidate wall-clock mean (>1 means candidate is faster)."""
- if candidate.wall_mean <= 0:
- return float("inf")
- return baseline.wall_mean / candidate.wall_mean
diff --git a/scripts/benchmarks/_mocks.py b/scripts/benchmarks/_mocks.py
deleted file mode 100644
index b6baed3..0000000
--- a/scripts/benchmarks/_mocks.py
+++ /dev/null
@@ -1,86 +0,0 @@
-"""HTTP mock infrastructure for deterministic concurrency benchmarks.
-
-The fli library talks to Google Flights through a single ``Client`` wrapper
-(`fli.search.client.Client`). We swap that out with a controlled-latency
-stub so we can measure how well the search code parallelises *its* work,
-without actually touching the network or Google's 10 req/sec ceiling.
-
-Each stub method:
-
-* Sleeps for a configurable number of milliseconds (simulated network RTT).
-* Returns a pre-baked ``Response``-shaped object whose ``.text`` is one of
- the captured fixtures.
-* Increments a thread-safe counter so the benchmark can verify the search
- layer actually issued the expected number of requests.
-"""
-
-from __future__ import annotations
-
-import threading
-import time
-from pathlib import Path
-from typing import Any
-
-FIXTURE_DIR = Path(__file__).resolve().parents[2] / "tests" / "search" / "fixtures"
-
-
-class _FakeResponse:
- """Minimal stand-in for ``curl_cffi.requests.Response``."""
-
- __slots__ = ("text", "status_code")
-
- def __init__(self, text: str, status_code: int = 200):
- self.text = text
- self.status_code = status_code
-
- def raise_for_status(self) -> None:
- if self.status_code >= 400:
- raise RuntimeError(f"HTTP {self.status_code}")
-
-
-class FakeClient:
- """Thread-safe controlled-latency HTTP stub.
-
- ``post()`` / ``get()`` block for ``latency_ms`` milliseconds and return
- the configured fixture body. The same fixture is returned on every
- call — search logic uses the response shape, not its content, for the
- parts we want to measure.
- """
-
- def __init__(
- self,
- fixture_text: str,
- *,
- latency_ms: float = 100.0,
- ):
- self._fixture = fixture_text
- self._latency_s = latency_ms / 1000.0
- self._lock = threading.Lock()
- self.calls = 0
- self.concurrent_high_water = 0
- self._in_flight = 0
-
- def _sleep(self) -> None:
- with self._lock:
- self.calls += 1
- self._in_flight += 1
- if self._in_flight > self.concurrent_high_water:
- self.concurrent_high_water = self._in_flight
- try:
- time.sleep(self._latency_s)
- finally:
- with self._lock:
- self._in_flight -= 1
-
- def post(self, url: str, **kwargs: Any) -> _FakeResponse:
- self._sleep()
- return _FakeResponse(self._fixture)
-
- def get(self, url: str, **kwargs: Any) -> _FakeResponse:
- self._sleep()
- return _FakeResponse(self._fixture)
-
-
-def load_fixture(name: str) -> str:
- """Read a captured response body from ``tests/search/fixtures/``."""
- return (FIXTURE_DIR / name).read_text()
diff --git a/scripts/benchmarks/bench.py b/scripts/benchmarks/bench.py
deleted file mode 100644
index e1935e2..0000000
--- a/scripts/benchmarks/bench.py
+++ /dev/null
@@ -1,498 +0,0 @@
-"""Comprehensive ``fli`` parsing & concurrency benchmark suite (30+ scenarios).
-
-Each scenario is deterministic — it replays a real or synthetic fixture
-through the same parser code production uses, with HTTP swapped for a
-controlled-latency stub. The output is grouped into five sections so
-regressions are easy to spot:
-
-1. **Parsing (CPU only)** — scales rows from 1 → 1000 to expose the
- per-row constant factor and any super-linear costs.
-2. **Wire-format reading** — single vs multi-chunk responses, the
- currency-token cache, booking decode at multiple vendor counts.
-3. **End-to-end search (I/O bound, mocked HTTP)** — one-way at three
- latencies, round-trip across ``top_n`` ∈ {2, 5, 10}, multi-city.
-4. **Date-range chunking** — 30 / 90 / 180 / 305-day ranges to verify
- parallelism scales with chunk count.
-5. **Concurrency primitives** — bare-metal numbers for
- ``parallel_map`` and ``TokenBucketRateLimiter``.
-
-Usage:
-
- uv run python scripts/benchmarks/bench.py [--iterations N] [--latency-ms M]
-"""
-
-from __future__ import annotations
-
-import argparse
-import sys
-import threading
-import time
-from copy import deepcopy
-from datetime import datetime, timedelta
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parents[2]
-sys.path.insert(0, str(ROOT))
-
-from fli.models import ( # noqa: E402
- Airport,
- DateSearchFilters,
- FlightSearchFilters,
- FlightSegment,
- PassengerInfo,
- TripType,
-)
-from fli.search import SearchDates, SearchFlights # noqa: E402
-from fli.search._concurrency import ( # noqa: E402
- TokenBucketRateLimiter,
- parallel_map,
-)
-from fli.search._decoders import parse_booking_chunk, parse_flight_row # noqa: E402
-from fli.search._wire import iter_wrb_chunks, parse_first_wrb_payload # noqa: E402
-from scripts.benchmarks._fixtures import ( # noqa: E402
- synthetic_booking_response,
- synthetic_date_response,
- synthetic_flight_response,
- synthetic_multi_chunk_flight_response,
-)
-from scripts.benchmarks._harness import ( # noqa: E402
- BenchResult,
- print_table,
- time_callable,
-)
-from scripts.benchmarks._mocks import FakeClient, load_fixture # noqa: E402
-
-# ---------------------------------------------------------------------------
-# Filter builders
-# ---------------------------------------------------------------------------
-
-
-def _today_plus(days: int) -> str:
- return (datetime.now() + timedelta(days=days)).strftime("%Y-%m-%d")
-
-
-def _one_way_filters() -> FlightSearchFilters:
- return FlightSearchFilters(
- trip_type=TripType.ONE_WAY,
- passenger_info=PassengerInfo(adults=1),
- flight_segments=[
- FlightSegment(
- departure_airport=[[Airport.JFK, 0]],
- arrival_airport=[[Airport.LAX, 0]],
- travel_date=_today_plus(30),
- )
- ],
- )
-
-
-def _round_trip_filters() -> FlightSearchFilters:
- return FlightSearchFilters(
- trip_type=TripType.ROUND_TRIP,
- passenger_info=PassengerInfo(adults=1),
- flight_segments=[
- FlightSegment(
- departure_airport=[[Airport.JFK, 0]],
- arrival_airport=[[Airport.LAX, 0]],
- travel_date=_today_plus(30),
- ),
- FlightSegment(
- departure_airport=[[Airport.LAX, 0]],
- arrival_airport=[[Airport.JFK, 0]],
- travel_date=_today_plus(37),
- ),
- ],
- )
-
-
-def _multi_city_filters(n_segments: int) -> FlightSearchFilters:
- cities = [Airport.JFK, Airport.LAX, Airport.SFO, Airport.ORD, Airport.MIA, Airport.BOS]
- segments = []
- for i in range(n_segments):
- dep = cities[i % len(cities)]
- arr = cities[(i + 1) % len(cities)]
- segments.append(
- FlightSegment(
- departure_airport=[[dep, 0]],
- arrival_airport=[[arr, 0]],
- travel_date=_today_plus(30 + i * 5),
- )
- )
- return FlightSearchFilters(
- trip_type=TripType.MULTI_CITY,
- passenger_info=PassengerInfo(adults=1),
- flight_segments=segments,
- )
-
-
-def _date_filters(days: int) -> DateSearchFilters:
- start = _today_plus(7)
- end = (datetime.strptime(start, "%Y-%m-%d") + timedelta(days=days - 1)).strftime("%Y-%m-%d")
- return DateSearchFilters(
- trip_type=TripType.ONE_WAY,
- passenger_info=PassengerInfo(adults=1),
- flight_segments=[
- FlightSegment(
- departure_airport=[[Airport.JFK, 0]],
- arrival_airport=[[Airport.LAX, 0]],
- travel_date=start,
- )
- ],
- from_date=start,
- to_date=end,
- )
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _rows_from_fixture(fixture: str) -> list:
- inner = parse_first_wrb_payload(fixture)
- if inner is None:
- return []
- return [item for i in (2, 3) if isinstance(inner[i], list) for item in inner[i][0]]
-
-
-def _wrap(meta: dict, fake: FakeClient) -> dict:
- meta["calls"] = fake.calls
- meta["concurrent_max"] = fake.concurrent_high_water
- return meta
-
-
-# ---------------------------------------------------------------------------
-# Section 1 — Parsing scaling (CPU only)
-# ---------------------------------------------------------------------------
-
-
-def parse_rows_scenario(name: str, rows: list, iterations: int) -> BenchResult:
- def run():
- return [parse_flight_row(r) for r in rows]
-
- res = time_callable(run, iterations=iterations, name=name)
- res.payload = {"rows": len(rows)}
- return res
-
-
-def bench_parsing_section(iters: int) -> list[BenchResult]:
- """Cover 1, 10, 50, 200, 1000 row volumes + multi-leg + diverse currencies."""
- results: list[BenchResult] = []
-
- # Real fixture
- real_body = load_fixture("flight_search_jfk_lax_oneway_usd.bin")
- real_rows = _rows_from_fixture(real_body)
- results.append(parse_rows_scenario("parse 1 row", real_rows[:1], iters * 40))
- results.append(parse_rows_scenario("parse 10 rows", real_rows[:10], iters * 20))
- results.append(parse_rows_scenario("parse 28 rows (real)", real_rows, iters * 10))
-
- # Synthetic
- syn_50 = _rows_from_fixture(synthetic_flight_response(rows=50))
- syn_200 = _rows_from_fixture(synthetic_flight_response(rows=200))
- syn_1000 = _rows_from_fixture(synthetic_flight_response(rows=1000))
- results.append(parse_rows_scenario("parse 50 rows", syn_50, iters * 4))
- results.append(parse_rows_scenario("parse 200 rows", syn_200, iters * 2))
- results.append(parse_rows_scenario("parse 1000 rows", syn_1000, iters))
-
- # Multi-leg (layover derivation)
- syn_ml = _rows_from_fixture(synthetic_flight_response(rows=50, multi_leg=True))
- results.append(parse_rows_scenario("parse 50 rows (multi-leg)", syn_ml, iters * 4))
-
- return results
-
-
-# ---------------------------------------------------------------------------
-# Section 2 — Wire-format reading
-# ---------------------------------------------------------------------------
-
-
-def bench_wire_section(iters: int) -> list[BenchResult]:
- """Wire decoder timings — single / multi chunk and booking parsing."""
- results: list[BenchResult] = []
-
- real = load_fixture("flight_search_jfk_lax_oneway_usd.bin")
- one_chunk = synthetic_multi_chunk_flight_response(rows_per_chunk=50, chunks=1)
- two_chunk = synthetic_multi_chunk_flight_response(rows_per_chunk=50, chunks=2)
- five_chunk = synthetic_multi_chunk_flight_response(rows_per_chunk=20, chunks=5)
-
- results.append(
- time_callable(
- lambda: parse_first_wrb_payload(real),
- iterations=iters * 20,
- name="iter_wrb: real 28-row body",
- )
- )
- results.append(
- time_callable(
- lambda: list(iter_wrb_chunks(one_chunk)),
- iterations=iters * 20,
- name="iter_wrb: synthetic 1 chunk",
- )
- )
- results.append(
- time_callable(
- lambda: list(iter_wrb_chunks(two_chunk)),
- iterations=iters * 10,
- name="iter_wrb: synthetic 2 chunks",
- )
- )
- results.append(
- time_callable(
- lambda: list(iter_wrb_chunks(five_chunk)),
- iterations=iters * 10,
- name="iter_wrb: synthetic 5 chunks",
- )
- )
-
- booking_small = synthetic_booking_response(vendor_count=5)
- booking_large = synthetic_booking_response(vendor_count=50)
- results.append(
- time_callable(
- lambda: parse_booking_chunk(parse_first_wrb_payload(booking_small)),
- iterations=iters * 10,
- name="parse_booking: 5 vendors",
- )
- )
- results.append(
- time_callable(
- lambda: parse_booking_chunk(parse_first_wrb_payload(booking_large)),
- iterations=iters * 5,
- name="parse_booking: 50 vendors",
- )
- )
-
- return results
-
-
-# ---------------------------------------------------------------------------
-# Section 3 — End-to-end search (mocked HTTP, varying latency)
-# ---------------------------------------------------------------------------
-
-
-def bench_search_section(iters: int) -> list[BenchResult]:
- """End-to-end search wall clock with mocked HTTP at varying latency."""
- fixture = load_fixture("flight_search_jfk_lax_oneway_usd.bin")
- results: list[BenchResult] = []
-
- def make_one_way(latency_ms: float):
- fake = FakeClient(fixture, latency_ms=latency_ms)
-
- def run():
- search = SearchFlights()
- search.client = fake
- return search.search(_one_way_filters())
-
- res = time_callable(run, iterations=iters, name=f"one-way @ {int(latency_ms)}ms")
- res.payload = _wrap({}, fake)
- return res
-
- def make_round_trip(top_n: int, latency_ms: float = 100.0):
- fake = FakeClient(fixture, latency_ms=latency_ms)
-
- def run():
- search = SearchFlights()
- search.client = fake
- return search.search(_round_trip_filters(), top_n=top_n)
-
- res = time_callable(
- run,
- iterations=iters,
- name=f"round-trip top_n={top_n} @ {int(latency_ms)}ms",
- )
- res.payload = _wrap({}, fake)
- return res
-
- def make_multi_city(n_segments: int, latency_ms: float = 100.0):
- fake = FakeClient(fixture, latency_ms=latency_ms)
-
- def run():
- search = SearchFlights()
- search.client = fake
- f = _multi_city_filters(n_segments)
- return search.search(f, top_n=3)
-
- res = time_callable(
- run,
- iterations=iters,
- name=f"multi-city {n_segments}-seg @ {int(latency_ms)}ms",
- )
- res.payload = _wrap({}, fake)
- return res
-
- results.append(make_one_way(50.0))
- results.append(make_one_way(120.0))
- results.append(make_one_way(500.0))
- results.append(make_round_trip(top_n=2))
- results.append(make_round_trip(top_n=5))
- results.append(make_round_trip(top_n=10))
- results.append(make_multi_city(n_segments=3))
-
- return results
-
-
-# ---------------------------------------------------------------------------
-# Section 4 — Date-range chunking
-# ---------------------------------------------------------------------------
-
-
-def bench_dates_section(iters: int) -> list[BenchResult]:
- """Date-range searches — chunk count grows linearly with range size."""
- date_body = synthetic_date_response(days=61)
- results: list[BenchResult] = []
-
- def make(days: int, latency_ms: float = 120.0):
- fake = FakeClient(date_body, latency_ms=latency_ms)
-
- def run():
- search = SearchDates()
- search.client = fake
- return search.search(deepcopy(_date_filters(days)))
-
- chunks = (days + 60) // 61
- res = time_callable(
- run,
- iterations=iters,
- name=f"dates {days}d ({chunks} chunks)",
- )
- res.payload = _wrap({}, fake)
- return res
-
- results.append(make(30)) # 1 chunk
- results.append(make(90)) # 2 chunks
- results.append(make(180)) # 3 chunks
- results.append(make(244)) # 4 chunks
- results.append(make(305)) # 5 chunks (max range)
-
- return results
-
-
-# ---------------------------------------------------------------------------
-# Section 5 — Concurrency primitives
-# ---------------------------------------------------------------------------
-
-
-def bench_concurrency_section(iters: int) -> list[BenchResult]:
- """Bare-metal numbers for the primitives the search code depends on."""
- results: list[BenchResult] = []
-
- # parallel_map: small list (5 items, 10ms each)
- def pmap_small():
- return parallel_map(lambda x: time.sleep(0.01) or x, list(range(5)), max_workers=5)
-
- results.append(time_callable(pmap_small, iterations=iters, name="parallel_map: 5x10ms"))
-
- # parallel_map: large list (20 items, 5ms each)
- def pmap_large():
- return parallel_map(lambda x: time.sleep(0.005) or x, list(range(20)), max_workers=10)
-
- results.append(time_callable(pmap_large, iterations=iters, name="parallel_map: 20x5ms"))
-
- # parallel_map: synchronous fast path
- def pmap_sync():
- return parallel_map(lambda x: x * 2, [42])
-
- results.append(
- time_callable(pmap_sync, iterations=iters * 100, name="parallel_map: 1 item (sync)")
- )
-
- # parallel_map: CPU work — token bucket overhead shouldn't matter
- def pmap_cpu():
- return parallel_map(lambda x: sum(range(x)), [10000] * 8, max_workers=4)
-
- results.append(time_callable(pmap_cpu, iterations=iters, name="parallel_map: 8 CPU jobs"))
-
- # TokenBucket: uncontended acquire
- bucket_uncon = TokenBucketRateLimiter(calls=1000, period=1.0)
- results.append(
- time_callable(
- lambda: bucket_uncon.acquire(),
- iterations=iters * 200,
- name="bucket acquire (uncontended)",
- )
- )
-
- # TokenBucket: contended (5 threads sharing 100/sec budget)
- bucket_con = TokenBucketRateLimiter(calls=100, period=1.0)
-
- def contended():
- threads = []
- for _ in range(5):
- t = threading.Thread(target=lambda: [bucket_con.acquire() for _ in range(4)])
- threads.append(t)
- t.start()
- for t in threads:
- t.join()
-
- results.append(
- time_callable(contended, iterations=iters, name="bucket: 5 threads x 4 acquires")
- )
-
- return results
-
-
-# ---------------------------------------------------------------------------
-# Entry point
-# ---------------------------------------------------------------------------
-
-
-def main() -> int:
- """Run all five sections and print a single combined report."""
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--iterations", type=int, default=5)
- parser.add_argument(
- "--latency-ms",
- type=float,
- default=120.0,
- help="Used wherever a benchmark doesn't fix its own latency.",
- )
- args = parser.parse_args()
- iters = args.iterations
- print(f"Running benchmark suite: {iters} iterations per scenario.\n")
-
- sections = [
- ("Section 1 — Parsing (CPU)", bench_parsing_section(iters)),
- ("Section 2 — Wire format", bench_wire_section(iters)),
- ("Section 3 — End-to-end search (mocked HTTP)", bench_search_section(iters)),
- ("Section 4 — Date-range chunking", bench_dates_section(iters)),
- ("Section 5 — Concurrency primitives", bench_concurrency_section(iters)),
- ]
-
- total = sum(len(r) for _, r in sections)
- print(f"Total scenarios: {total}\n")
- for title, results in sections:
- print_table(title, results)
-
- # Concurrency observations for the search/date sections.
- print("\nObserved request concurrency:")
- for title, results in sections:
- if "Section 3" in title or "Section 4" in title:
- for r in results:
- meta = r.payload or {}
- calls = meta.get("calls", 0) / max(r.iterations, 1)
- peak = meta.get("concurrent_max", 1)
- print(f" {r.name:40s} calls/run={calls:5.1f} peak in-flight={peak}")
-
- # Top-level throughput summary.
- print("\nKey throughput metrics:")
- for title, results in sections:
- if "Section 1" in title:
- for r in results:
- rows = (r.payload or {}).get("rows", 0)
- if rows and r.wall_mean > 0:
- throughput = rows * 1000.0 / r.wall_mean
- print(f" {r.name:30s} {throughput:>12,.0f} rows/sec")
-
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
-
-
-# Re-exports for the comparison driver (compare.py).
-__all__ = [
- "bench_concurrency_section",
- "bench_dates_section",
- "bench_parsing_section",
- "bench_search_section",
- "bench_wire_section",
-]
diff --git a/scripts/benchmarks/compare.py b/scripts/benchmarks/compare.py
deleted file mode 100644
index 0ee79c9..0000000
--- a/scripts/benchmarks/compare.py
+++ /dev/null
@@ -1,130 +0,0 @@
-"""Sequential-vs-parallel comparison driver across every scenario in ``bench.py``.
-
-Runs each scenario twice — once with the shared executor forced down to a
-single worker (so ``parallel_map`` falls back to its synchronous loop),
-once with the default capacity — and prints a side-by-side table with
-speedup factors.
-
-Scenarios where parallelism makes no difference (pure CPU parsing,
-single-chunk wire reads) still appear so we can confirm that the
-parallel mode never *regresses* the fast path.
-
-Usage:
-
- uv run python scripts/benchmarks/compare.py [--iterations N]
-"""
-
-from __future__ import annotations
-
-import argparse
-import sys
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parents[2]
-sys.path.insert(0, str(ROOT))
-
-# Force-toggle the shared executor before importing any scenario builders
-# so they all see the configured worker cap.
-import fli.search._concurrency as _conc # noqa: E402
-from scripts.benchmarks._harness import BenchResult, speedup # noqa: E402
-from scripts.benchmarks.bench import ( # noqa: E402
- bench_concurrency_section,
- bench_dates_section,
- bench_parsing_section,
- bench_search_section,
- bench_wire_section,
-)
-
-
-def _set_parallel(enabled: bool) -> None:
- _conc.shutdown_executor()
- _conc.configure_concurrency(10 if enabled else 1)
-
-
-def _collect(iters: int) -> dict[str, list[BenchResult]]:
- return {
- "Parsing (CPU)": bench_parsing_section(iters),
- "Wire format": bench_wire_section(iters),
- "End-to-end search": bench_search_section(iters),
- "Date-range chunking": bench_dates_section(iters),
- "Concurrency primitives": bench_concurrency_section(iters),
- }
-
-
-def _print_section(title: str, seq: list[BenchResult], par: list[BenchResult]) -> None:
- print(f"\n{title}")
- print("-" * len(title))
- header = ("scenario", "seq ms", "par ms", "speedup", "saved ms")
- rows: list[tuple[str, ...]] = []
- for s, p in zip(seq, par, strict=False):
- sup = speedup(s, p)
- saved = s.wall_mean - p.wall_mean
- rows.append(
- (
- s.name,
- f"{s.wall_mean:8.2f}",
- f"{p.wall_mean:8.2f}",
- f"{sup:5.2f}x",
- f"{saved:+8.2f}",
- )
- )
- widths = [max(len(header[i]), *(len(r[i]) for r in rows)) for i in range(len(header))]
- print(" ".join(h.ljust(widths[i]) for i, h in enumerate(header)))
- print(" ".join("-" * w for w in widths))
- for row in rows:
- print(" ".join(c.ljust(widths[i]) for i, c in enumerate(row)))
-
-
-def _print_top_speedups(all_seq: list[BenchResult], all_par: list[BenchResult]) -> None:
- pairs = [(s, p, speedup(s, p)) for s, p in zip(all_seq, all_par, strict=False)]
- pairs.sort(key=lambda x: x[2], reverse=True)
- print("\nTop speedups overall:")
- print("-" * 22)
- for s, p, sup in pairs[:8]:
- print(
- f" {s.name:45s} {s.wall_mean:7.2f}ms → {p.wall_mean:7.2f}ms "
- f"({sup:5.2f}x, saved {s.wall_mean - p.wall_mean:+7.2f}ms)"
- )
-
-
-def main() -> int:
- """Run every scenario in sequential mode, then parallel, then diff."""
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--iterations", type=int, default=3)
- args = parser.parse_args()
-
- iters = args.iterations
- print(f"Comparison run: {iters} iterations per scenario, parallel toggled via executor cap.\n")
-
- _set_parallel(False)
- print("[1/2] Running SEQUENTIAL (max_workers=1)...")
- seq_sections = _collect(iters)
-
- _set_parallel(True)
- print("[2/2] Running PARALLEL (max_workers=10)...")
- par_sections = _collect(iters)
-
- flat_seq: list[BenchResult] = []
- flat_par: list[BenchResult] = []
- for title, seq in seq_sections.items():
- par = par_sections[title]
- _print_section(title, seq, par)
- flat_seq.extend(seq)
- flat_par.extend(par)
-
- _print_top_speedups(flat_seq, flat_par)
-
- # Quick interpretive summary.
- total_seq = sum(s.wall_mean for s in flat_seq)
- total_par = sum(p.wall_mean for p in flat_par)
- print(
- f"\nAggregate across {len(flat_seq)} scenarios: "
- f"sequential={total_seq:.0f}ms parallel={total_par:.0f}ms "
- f"saved={total_seq - total_par:.0f}ms ({total_seq / max(total_par, 1e-6):.2f}x)"
- )
-
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())