π The fast, Pythonic way to build MCP servers and clients.
fix(transforms): bound Jev search narrowing (#5199)
nate nowack345647dafe9e
diff --git a/docs/servers/transforms/tool-search.mdx b/docs/servers/transforms/tool-search.mdxindex f5fc64ad7..815bea09e 100644--- a/docs/servers/transforms/tool-search.mdx+++ b/docs/servers/transforms/tool-search.mdx@@ -134,7 +134,7 @@ A catalog no larger than `3 * shortlist` skips the wide pass.| `model` | `jev-latest` | TypeSafe model. Pin a versioned id once `fit_threshold` is tuned. || `api_key` | `TYPESAFE_API_KEY` | TypeSafe API key. || `client` | none | A ready `AsyncTypeSafeClient` to use instead of building one. |-| `shortlist` | 8 | Candidates each wide-pass request carries forward; the close read sees at most three times this. |+| `shortlist` | 8 | Candidates each wide-pass request carries forward; with `close_read=True`, it must be at most half of `chunk_size` so repeated passes remain bounded. The close read sees at most three times this or 255 tools, whichever is smaller. || `fit_threshold` | 0.3 | Minimum "does this tool do it" probability for a tool to be returned. || `chunk_size` | 150 | Tools per wide-pass request, at most 255. || `close_read` | `True` | Re-read the shortlist with full descriptions in a second request. `False` asks the fit question of every tool in the wide pass instead: one round trip, summaries only. On the 187-tool Prefect catalog it was 0.06s faster and six points worse at returning the right tool first. |diff --git a/fastmcp_slim/fastmcp/experimental/transforms/jev_search.py b/fastmcp_slim/fastmcp/experimental/transforms/jev_search.pyindex c8d34a54b..5beefdaec 100644--- a/fastmcp_slim/fastmcp/experimental/transforms/jev_search.py+++ b/fastmcp_slim/fastmcp/experimental/transforms/jev_search.py@@ -106,9 +106,10 @@ class JevSearchTransform(BaseSearchTransform):timeout: Seconds per API attempt when the transform builds its ownclient. A search is one to a few requests.shortlist: How many candidates each wide-pass request carries- forward. The close read sees at most ``3 * shortlist``- candidates; a larger catalog is narrowed with further wide- passes first.+ forward. With ``close_read=True``, this must be at most half of+ ``chunk_size`` so every wide pass reduces the candidate set by a+ meaningful amount. The close read sees at most+ ``min(3 * shortlist, 255)`` candidates.fit_threshold: A candidate whose "does this tool do what the requestasks" probability falls below this is dropped from the results.Tune it against queries from your own users.@@ -155,6 +156,10 @@ class JevSearchTransform(BaseSearchTransform):raise ValueError("shortlist must be at least 1")if not 1 <= chunk_size <= MAX_CHOICE_OPTIONS:raise ValueError(f"chunk_size must be between 1 and {MAX_CHOICE_OPTIONS}")+ if close_read and shortlist * 2 > chunk_size:+ raise ValueError(+ "shortlist must be at most half of chunk_size when close_read is enabled"+ )if not 0 <= fit_threshold <= 1:raise ValueError("fit_threshold must be between 0 and 1")if summary_chars < 8 or detail_chars < 8:@@ -172,7 +177,7 @@ class JevSearchTransform(BaseSearchTransform):self._client = clientself._client_lock: asyncio.Lock | None = Noneself._shortlist = shortlist- self._max_candidates = 3 * shortlist+ self._max_candidates = min(3 * shortlist, MAX_CHOICE_OPTIONS)self._fit_threshold = fit_thresholdself._close_read = close_readself._chunk_size = chunk_sizediff --git a/tests/experimental/transforms/test_jev_search.py b/tests/experimental/transforms/test_jev_search.pyindex 944587942..e727d66ce 100644--- a/tests/experimental/transforms/test_jev_search.py+++ b/tests/experimental/transforms/test_jev_search.py@@ -17,6 +17,7 @@ import pytestfrom fastmcp import Client, FastMCPfrom fastmcp.experimental.transforms.jev_search import (+ MAX_CHOICE_OPTIONS,RERANK_INSTRUCTIONS,WIDE_INSTRUCTIONS,JevSearchTransform,@@ -215,7 +216,7 @@ class TestLargeCatalog:mcp = _server(*names)# close read holds 3 * shortlist = 6; seven tools need a wide passmcp.add_transform(- JevSearchTransform(client=jev, shortlist=2, chunk_size=3, max_results=3)+ JevSearchTransform(client=jev, shortlist=2, chunk_size=4, max_results=3))assert await _search(mcp, "the sixth thing") == [@@ -227,21 +228,19 @@ class TestLargeCatalog:wide = jev.requests[:-1]close = jev.requests[-1]assert [sorted(r["questions"]["which"]["criteria"]) for r in wide] == [- ["tool_00", "tool_01", "tool_02"],- ["tool_03", "tool_04", "tool_05"],- ["tool_06"],+ ["tool_00", "tool_01", "tool_02", "tool_03"],+ ["tool_04", "tool_05", "tool_06"],]assert all(r["questions"]["which"]["instructions"] == WIDE_INSTRUCTIONS for r in wide)assert all(len(r["questions"]) == 1 for r in wide)assert all(r["state"] == {"request": "the sixth thing"} for r in wide)- # two per chunk survive the wide pass; the third chunk only has one+ # two per chunk survive the wide passassert sorted(close["questions"]["which"]["criteria"]) == ["tool_01","tool_02","tool_04",- "tool_05","tool_06",]@@ -258,6 +257,20 @@ class TestLargeCatalog:close = jev.requests[-1]assert len(close["questions"]["which"]["criteria"]) == 6+ async def test_close_read_never_exceeds_choice_limit(self):+ names = [f"t{i:03d}" for i in range(MAX_CHOICE_OPTIONS + 1)]+ jev = FakeJev()+ mcp = _server(*names)+ mcp.add_transform(JevSearchTransform(client=jev, shortlist=100, chunk_size=200))++ await _search(mcp, "x")++ choice_sizes = [+ len(request["questions"]["which"]["criteria"]) for request in jev.requests+ ]+ assert choice_sizes == [200, 56, 156]+ assert max(choice_sizes) <= MAX_CHOICE_OPTIONS+async def test_wide_pass_uses_first_paragraph_only(self):jev = FakeJev()mcp = _server(*(f"t{i}" for i in range(4)))@@ -381,6 +394,11 @@ class TestConfiguration:JevSearchTransform(client=FakeJev(), fit_threshold=1.5)with pytest.raises(ValueError):JevSearchTransform(client=FakeJev(), shortlist=0)+ with pytest.raises(ValueError, match="at most half"):+ JevSearchTransform(client=FakeJev(), shortlist=254, chunk_size=255)+ JevSearchTransform(+ client=FakeJev(), close_read=False, shortlist=8, chunk_size=5+ )with pytest.raises(ValueError, match="255"):JevSearchTransform(client=FakeJev(), chunk_size=300)with pytest.raises(ValueError):