Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
refactor(test): derive offline gateway e2e cache from the kit deck (#947)
Louis Choquel8244be5999cc
diff --git a/tests/e2e/agent_cli/conftest.py b/tests/e2e/agent_cli/conftest.pyindex 96d69cf62..a80bae9e2 100644--- a/tests/e2e/agent_cli/conftest.py+++ b/tests/e2e/agent_cli/conftest.py@@ -17,12 +17,28 @@ import jsonimport shutilfrom datetime import datetime, timezonefrom pathlib import Path-from typing import Any+from typing import TYPE_CHECKING, Anyimport pytest+from pipelex.cogt.model_backends.backend import PipelexBackend+from pipelex.cogt.model_backends.model_type import ModelType+from pipelex.cogt.models.exceptions import ModelReferenceParseError+from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprint+from pipelex.cogt.models.model_manager import ModelManager+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKindfrom pipelex.kit.paths import get_kit_configs_dirfrom pipelex.system.pipelex_service.remote_config_cache import CACHE_SCHEMA_VERSION+from pipelex.tools.misc.toml_utils import load_toml_from_path++if TYPE_CHECKING:+ from pipelex.cogt.models.model_deck import (+ ExtractDeckBlueprint,+ ImgGenDeckBlueprint,+ LLMDeckBlueprint,+ ModelDeckBlueprint,+ SearchDeckBlueprint,+ )REPO_ROOT = Path(__file__).resolve().parents[3]PIPELEX_AGENT_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex-agent"@@ -31,6 +47,108 @@ OFFLINE_BUNDLES_DIR = REPO_ROOT / "tests" / "e2e" / "data" / "offline_mode"# Reserved/unroutable address — httpx will raise ConnectError immediately without hanging.UNREACHABLE_REMOTE_CONFIG_URL = "http://127.0.0.1:1/pipelex_remote_config.json"+# Minimal-but-valid gateway spec bodies per model type (valid against InferenceModelSpecBlueprint).+# Each ``sdk`` names the dedicated gateway worker for its type — gateway_completions (LLM),+# gateway_img_gen (image), gateway_extract, gateway_search — i.e. a value the matching worker factory's+# gateway branch accepts (an unrecognized sdk like "gateway_image" hits ``case _: raise NotImplementedError``).+# The sdk is consumed only at worker creation, which the offline dry-run tests never reach+# (ContentGeneratorDry mocks generation), so the membership check is indifferent to it — but it must still+# be real so the fake cache stays faithful and any non-dry consumer of this helper can build a worker.+_GATEWAY_SPEC_TEMPLATE_BY_TYPE: dict[ModelType, dict[str, Any]] = {+ ModelType.LLM: {"sdk": "gateway_completions", "model_type": "llm", "inputs": ["text"], "outputs": ["text", "structured"]},+ ModelType.IMG_GEN: {"sdk": "gateway_img_gen", "model_type": "img_gen"},+ ModelType.TEXT_EXTRACTOR: {"sdk": "gateway_extract", "model_type": "text_extractor"},+ ModelType.SEARCH: {"sdk": "gateway_search", "model_type": "search"},+}+++def _deck_section_handle_references(+ section: LLMDeckBlueprint | ExtractDeckBlueprint | ImgGenDeckBlueprint | SearchDeckBlueprint,+) -> list[str]:+ """Every raw model reference a deck section names via aliases, waterfalls, or presets.++ Choice defaults are intentionally NOT read here: in the kit deck they are always+ ``@alias`` / ``$preset`` references, so the terminal handles they resolve to are+ already named as alias targets / preset models and get collected anyway. Skipping+ them keeps this free of the ``disabled`` sentinel and the typed choice-default union.+ """+ raw_references: list[str] = list(section.aliases.values())+ for waterfall_entries in section.waterfalls.values():+ raw_references.extend(waterfall_entries)+ for preset in section.presets.values():+ raw_references.append(preset.model)+ return raw_references+++def load_kit_model_deck_blueprint() -> ModelDeckBlueprint:+ """Load the shipped kit deck blueprint — the same deck files the offline subprocess loads."""+ deck_dir = Path(str(get_kit_configs_dir())) / "inference" / "deck"+ return load_model_deck_blueprint(model_deck_paths=ModelManager.get_model_deck_paths(deck_dir_path=str(deck_dir)))+++def software_only_internal_handles() -> set[str]:+ """Handles served by the local ``internal`` backend — the ones the Pipelex Gateway never provides.++ ``PipelexBackend.INTERNAL`` is the software-only backend ("runs internally, without AI" — e.g. the+ pypdfium2 / docling text extractors). Those handles have no gateway equivalent, so they must be+ excluded from the derived gateway specs: otherwise the primed cache would claim the gateway serves a+ software-only extractor, and a deck alias like ``@default-no-inference`` / ``@default-text-from-pdf``+ could resolve through the fake ``gateway_extract`` worker instead of the real internal backend.++ Read straight from the shipped backend spec file (no booted Pipelex needed) and keyed off the enum+ value, so adding a new software-only extractor to the internal backend auto-excludes it here too.+ Provider-backed models (claude, gpt, linkup, nano-banana, ...) are intentionally NOT excluded: the+ gateway genuinely proxies those, so they belong in the faithful gateway cache.+ """+ internal_spec_path = Path(str(get_kit_configs_dir())) / "inference" / "backends" / f"{PipelexBackend.INTERNAL}.toml"+ raw_specs = load_toml_from_path(path=str(internal_spec_path))+ return {name for name, spec in raw_specs.items() if name != "defaults" and isinstance(spec, dict)}+++def gateway_backend_model_specs_for_kit_deck() -> dict[str, Any]:+ """Build a gateway ``backend_model_specs`` payload covering every concrete model handle the kit deck names.++ Derived from the shipped kit deck rather than a hand-maintained list, so the primed+ offline cache auto-tracks deck changes — e.g. promoting a premium alias to a new model+ no longer silently breaks these tests. Every bare handle named in the deck's aliases,+ waterfalls, and presets (across all model types) gets a minimal spec declaring the+ gateway sdk for its type — all ``ModelManager._enforce_gateway_model_membership`` needs+ is name membership. Deck/gateway consistency itself is covered by ``TestModelDeckReferences``.++ Handles served by the software-only ``internal`` backend (see ``software_only_internal_handles``)+ are excluded: the gateway never provides them, so claiming it does would make the fake cache+ unfaithful and let a software-only extractor alias resolve through the gateway worker. Those+ handles are still reachable in the offline subprocess via the local internal backend, so the+ membership check passes for them without a gateway spec.+ """+ blueprint = load_kit_model_deck_blueprint()+ excluded_handles = software_only_internal_handles()++ handle_model_types: dict[str, ModelType] = {}+ for model_type, section in (+ (ModelType.LLM, blueprint.llm),+ (ModelType.TEXT_EXTRACTOR, blueprint.extract),+ (ModelType.IMG_GEN, blueprint.img_gen),+ (ModelType.SEARCH, blueprint.search),+ ):+ for raw_reference in _deck_section_handle_references(section):+ try:+ ref = ModelReference.parse(raw_reference)+ except ModelReferenceParseError:+ continue+ match ref.kind:+ case ModelReferenceKind.HANDLE:+ if ref.name in excluded_handles:+ continue+ handle_model_types.setdefault(ref.name, model_type)+ case ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL | ModelReferenceKind.PRESET:+ continue++ specs: dict[str, Any] = {"defaults": {"sdk": "gateway_completions"}}+ for handle, model_type in handle_model_types.items():+ specs[handle] = dict(_GATEWAY_SPEC_TEMPLATE_BY_TYPE[model_type])+ return specs+def _copy_kit_configs_into(pipelex_dir: Path) -> None:"""Mirror the kit's ``configs/`` tree into ``pipelex_dir`` so the subprocess can boot.@@ -196,7 +314,10 @@ __all__ = ["OFFLINE_BUNDLES_DIR","PIPELEX_AGENT_BIN","UNREACHABLE_REMOTE_CONFIG_URL",+ "gateway_backend_model_specs_for_kit_deck",+ "load_kit_model_deck_blueprint","set_gateway_enabled",+ "software_only_internal_handles","write_active_routing_profile","write_remote_config_cache",]diff --git a/tests/e2e/agent_cli/test_offline_run_dry.py b/tests/e2e/agent_cli/test_offline_run_dry.pyindex a9080bfaf..5f4597934 100644--- a/tests/e2e/agent_cli/test_offline_run_dry.py+++ b/tests/e2e/agent_cli/test_offline_run_dry.py@@ -19,6 +19,8 @@ import pytestfrom tests.e2e.agent_cli.conftest import (OFFLINE_BUNDLES_DIR,PIPELEX_AGENT_BIN,+ gateway_backend_model_specs_for_kit_deck,+ load_kit_model_deck_blueprint,set_gateway_enabled,write_active_routing_profile,write_remote_config_cache,@@ -104,81 +106,14 @@ def _parse_agent_json(output: str) -> dict[str, object]:raise AssertionError(msg)-# A minimal but structurally complete cached remote-config payload. ``backend_model_specs``-# must include every terminal handle referenced by the default kit deck so the gateway-# membership check passes for the "known-model" scenario.-def _comprehensive_cached_backend_model_specs() -> dict[str, object]:- """Build a ``backend_model_specs`` payload that satisfies the default kit deck.+def _cached_remote_config_payload() -> dict[str, object]:+ """A structurally complete cached remote-config payload for the primed-cache scenarios.- The kit's ``1_llm_deck.toml`` references many concrete handles via aliases/waterfalls.- We list them all here so the gateway-membership check in ``Pipelex.setup`` accepts the- cache. The spec body uses ``gateway_completions`` (LLMs) / ``gateway_image`` (image-gen)- sdks since the gateway proxies to those — values are minimal but valid against- ``InferenceModelSpecBlueprint``.+ ``backend_model_specs`` is derived from the shipped kit deck (see+ ``gateway_backend_model_specs_for_kit_deck``) so the gateway-membership check in+ ``Pipelex.setup`` accepts the cache without a hand-maintained handle list that goes+ stale whenever the deck promotes an alias to a new model."""- llm_handle_template: dict[str, object] = {- "sdk": "gateway_completions",- "model_type": "llm",- "inputs": ["text"],- "outputs": ["text", "structured"],- }- img_gen_handle_template: dict[str, object] = {- "sdk": "gateway_image",- "model_type": "img_gen",- }- extract_handle_template: dict[str, object] = {- "sdk": "gateway_extract",- "model_type": "text_extractor",- }- search_handle_template: dict[str, object] = {- "sdk": "gateway_search",- "model_type": "search",- }-- llm_handles = [- "claude-3.7-sonnet",- "claude-4.5-haiku",- "claude-4.5-sonnet",- "claude-4.6-opus",- "claude-4.6-sonnet",- "claude-4.7-opus",- "claude-4.8-opus",- "gemini-flash-latest",- "gemini-flash-lite-latest",- "gemini-pro-latest",- "gpt-4o",- "gpt-4o-mini",- "gpt-5",- "gpt-5-mini",- "gpt-5-nano",- "gpt-5.1",- "gpt-5.2",- "mistral-large",- ]- img_gen_handles = [- "gpt-image-1",- "gpt-image-1-mini",- "gpt-image-2",- "nano-banana",- "nano-banana-2",- "nano-banana-pro",- ]- extract_handles = ["azure-document-intelligence", "linkup-fetch"]- search_handles = ["linkup-standard", "linkup-deep"]-- specs: dict[str, object] = {"defaults": {"sdk": "gateway_completions"}}- for handle in llm_handles:- specs[handle] = dict(llm_handle_template)- for handle in img_gen_handles:- specs[handle] = dict(img_gen_handle_template)- for handle in extract_handles:- specs[handle] = dict(extract_handle_template)- for handle in search_handles:- specs[handle] = dict(search_handle_template)- return specs---def _cached_remote_config_payload() -> dict[str, object]:return {"posthog": {"project_api_key": "test-key",@@ -186,7 +121,7 @@ def _cached_remote_config_payload() -> dict[str, object]:"is_geoip_enabled": False,"is_debug_enabled": False,},- "backend_model_specs": _comprehensive_cached_backend_model_specs(),+ "backend_model_specs": gateway_backend_model_specs_for_kit_deck(),"aws_region": "us-east-1",}@@ -253,6 +188,44 @@ class TestOfflineDryRun:message = str(payload.get("message", ""))assert "pipelex init" in message.lower(), f"Error must mention `pipelex init` remediation; got: {message!r}"+ def test_primed_cache_specs_track_kit_deck_premium_alias(self) -> None:+ """Regression: the primed gateway cache is derived from the kit deck, so it tracks alias promotions.++ Reads the deck's own ``default-premium`` target — whatever model it currently points+ to — and asserts the derived gateway specs cover it, without hardcoding the model name.+ If a future deck promotes the premium alias to a new handle this keeps passing; if the+ derivation ever stops covering a referenced handle it fails fast here (a unit-speed+ check) instead of deep inside a subprocess setup with an opaque GatewayUnknownModelError.+ """+ blueprint = load_kit_model_deck_blueprint()+ premium_target = blueprint.llm.aliases["default-premium"]+ specs = gateway_backend_model_specs_for_kit_deck()+ assert premium_target in specs, (+ f"Derived gateway specs must cover the kit deck's default-premium target '{premium_target}'; got handles: {sorted(specs)}"+ )++ def test_primed_cache_excludes_software_only_internal_handles(self) -> None:+ """Regression: the gateway cache must not claim software-only internal extractors as gateway models.++ ``@default-no-inference`` / ``@default-text-from-pdf`` resolve to an extractor served by the+ local ``internal`` backend (``PipelexBackend.INTERNAL`` — "runs internally, without AI"), which+ the Pipelex Gateway never provides. If the derived gateway specs claimed it, an offline dry-run+ could resolve the alias to the fake ``gateway_extract`` worker instead of exercising the real+ internal backend. The companion assertion guards the other direction: a genuinely gateway-served+ extract model (the ``default-extract-document`` target) must stay covered, so the exclusion does+ not over-reach onto provider-backed models the gateway does proxy.+ """+ blueprint = load_kit_model_deck_blueprint()+ software_only_target = blueprint.extract.aliases["default-no-inference"]+ gateway_extract_target = blueprint.extract.aliases["default-extract-document"]+ specs = gateway_backend_model_specs_for_kit_deck()+ assert software_only_target not in specs, (+ f"Derived gateway specs must exclude software-only internal handle '{software_only_target}'; got handles: {sorted(specs)}"+ )+ assert gateway_extract_target in specs, (+ f"Gateway-served extract target '{gateway_extract_target}' must remain covered; got handles: {sorted(specs)}"+ )+def test_gateway_known_with_cache_succeeds_offline(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:"""Gateway enabled + no network + primed cache → dry-run exits 0 with stale-cache warning."""pipelex_dir = hermetic_home / ".pipelex"@@ -270,6 +243,32 @@ class TestOfflineDryRun:f"Cached-source setup must surface a RemoteConfigStale warning on the envelope; got: {payload!r}")+ def test_gateway_img_gen_with_cache_succeeds_offline(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:+ """Gateway enabled + no network + primed cache → an image-gen pipe dry-runs offline.++ Companion to ``test_gateway_known_with_cache_succeeds_offline``, but the bundle's pipe is a+ ``PipeImgGen`` referencing a gateway image handle (``nano-banana``). It proves the derived+ cache covers img_gen handles too — not just LLMs — so the gateway-membership check passes for+ an image model and the offline dry-run completes.++ Scope note: dry-run mocks image generation via ``ContentGeneratorDry``, so this does NOT build+ the img-gen worker and therefore does not exercise the spec's ``sdk`` -> worker-factory mapping;+ it covers membership + handle resolution for image models. The sdk-vs-factory contract is a+ runtime (non-dry) concern.+ """+ pipelex_dir = hermetic_home / ".pipelex"+ set_gateway_enabled(pipelex_dir / "inference" / "backends.toml", enabled=True)+ write_remote_config_cache(pipelex_dir, _cached_remote_config_payload())++ staged_bundle = _stage_bundle(OFFLINE_BUNDLES_DIR / "gateway_img_gen_model", hermetic_home)+ result = _run_agent_bundle(staged_bundle, offline_subprocess_env, cwd=hermetic_home)++ assert result.returncode == 0, (+ f"Gateway img-gen dry-run with primed cache must succeed offline.\nstdout={result.stdout!r}\nstderr={result.stderr!r}"+ )+ payload = _parse_agent_json(result.stdout)+ assert "error" not in payload, payload+def test_gateway_unknown_with_cache_fails_with_clear_error(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:"""Bundle pipe references a model absent from the cached gateway specs → clear error.diff --git a/tests/e2e/data/offline_mode/gateway_img_gen_model/gateway_img_gen_model.mthds b/tests/e2e/data/offline_mode/gateway_img_gen_model/gateway_img_gen_model.mthdsnew file mode 100644index 000000000..0b5a76720--- /dev/null+++ b/tests/e2e/data/offline_mode/gateway_img_gen_model/gateway_img_gen_model.mthds@@ -0,0 +1,11 @@+domain = "offline_gateway_img_gen"+description = "Pipeline referencing a gateway image-generation model that exists in the cached gateway specs"+main_pipe = "gateway_img_gen_pipe"++[pipe]+[pipe.gateway_img_gen_pipe]+type = "PipeImgGen"+description = "Single image-generation step using a gateway-listed image model handle"+output = "Image"+model = "nano-banana"+prompt = "A single red circle on a white background."