Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

refactor(test): derive offline gateway e2e cache from the kit deck (#947)

Louis Choquel8244be5999cc
diff --git a/tests/e2e/agent_cli/conftest.py b/tests/e2e/agent_cli/conftest.py
index 96d69cf62..a80bae9e2 100644
--- a/tests/e2e/agent_cli/conftest.py
+++ b/tests/e2e/agent_cli/conftest.py
@@ -17,12 +17,28 @@ import json
import shutil
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pytest
+from pipelex.cogt.model_backends.backend import PipelexBackend
+from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.cogt.models.exceptions import ModelReferenceParseError
+from pipelex.cogt.models.model_deck_loader import load_model_deck_blueprint
+from pipelex.cogt.models.model_manager import ModelManager
+from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind
from pipelex.kit.paths import get_kit_configs_dir
from pipelex.system.pipelex_service.remote_config_cache import CACHE_SCHEMA_VERSION
+from pipelex.tools.misc.toml_utils import load_toml_from_path
+
+if TYPE_CHECKING:
+ from pipelex.cogt.models.model_deck import (
+ ExtractDeckBlueprint,
+ ImgGenDeckBlueprint,
+ LLMDeckBlueprint,
+ ModelDeckBlueprint,
+ SearchDeckBlueprint,
+ )
REPO_ROOT = Path(__file__).resolve().parents[3]
PIPELEX_AGENT_BIN = REPO_ROOT / ".venv" / "bin" / "pipelex-agent"
@@ -31,6 +47,108 @@ OFFLINE_BUNDLES_DIR = REPO_ROOT / "tests" / "e2e" / "data" / "offline_mode"
# Reserved/unroutable address — httpx will raise ConnectError immediately without hanging.
UNREACHABLE_REMOTE_CONFIG_URL = "http://127.0.0.1:1/pipelex_remote_config.json"
+# Minimal-but-valid gateway spec bodies per model type (valid against InferenceModelSpecBlueprint).
+# Each ``sdk`` names the dedicated gateway worker for its type — gateway_completions (LLM),
+# gateway_img_gen (image), gateway_extract, gateway_search — i.e. a value the matching worker factory's
+# gateway branch accepts (an unrecognized sdk like "gateway_image" hits ``case _: raise NotImplementedError``).
+# The sdk is consumed only at worker creation, which the offline dry-run tests never reach
+# (ContentGeneratorDry mocks generation), so the membership check is indifferent to it — but it must still
+# be real so the fake cache stays faithful and any non-dry consumer of this helper can build a worker.
+_GATEWAY_SPEC_TEMPLATE_BY_TYPE: dict[ModelType, dict[str, Any]] = {
+ ModelType.LLM: {"sdk": "gateway_completions", "model_type": "llm", "inputs": ["text"], "outputs": ["text", "structured"]},
+ ModelType.IMG_GEN: {"sdk": "gateway_img_gen", "model_type": "img_gen"},
+ ModelType.TEXT_EXTRACTOR: {"sdk": "gateway_extract", "model_type": "text_extractor"},
+ ModelType.SEARCH: {"sdk": "gateway_search", "model_type": "search"},
+}
+
+
+def _deck_section_handle_references(
+ section: LLMDeckBlueprint | ExtractDeckBlueprint | ImgGenDeckBlueprint | SearchDeckBlueprint,
+) -> list[str]:
+ """Every raw model reference a deck section names via aliases, waterfalls, or presets.
+
+ Choice defaults are intentionally NOT read here: in the kit deck they are always
+ ``@alias`` / ``$preset`` references, so the terminal handles they resolve to are
+ already named as alias targets / preset models and get collected anyway. Skipping
+ them keeps this free of the ``disabled`` sentinel and the typed choice-default union.
+ """
+ raw_references: list[str] = list(section.aliases.values())
+ for waterfall_entries in section.waterfalls.values():
+ raw_references.extend(waterfall_entries)
+ for preset in section.presets.values():
+ raw_references.append(preset.model)
+ return raw_references
+
+
+def load_kit_model_deck_blueprint() -> ModelDeckBlueprint:
+ """Load the shipped kit deck blueprint — the same deck files the offline subprocess loads."""
+ deck_dir = Path(str(get_kit_configs_dir())) / "inference" / "deck"
+ return load_model_deck_blueprint(model_deck_paths=ModelManager.get_model_deck_paths(deck_dir_path=str(deck_dir)))
+
+
+def software_only_internal_handles() -> set[str]:
+ """Handles served by the local ``internal`` backend — the ones the Pipelex Gateway never provides.
+
+ ``PipelexBackend.INTERNAL`` is the software-only backend ("runs internally, without AI" — e.g. the
+ pypdfium2 / docling text extractors). Those handles have no gateway equivalent, so they must be
+ excluded from the derived gateway specs: otherwise the primed cache would claim the gateway serves a
+ software-only extractor, and a deck alias like ``@default-no-inference`` / ``@default-text-from-pdf``
+ could resolve through the fake ``gateway_extract`` worker instead of the real internal backend.
+
+ Read straight from the shipped backend spec file (no booted Pipelex needed) and keyed off the enum
+ value, so adding a new software-only extractor to the internal backend auto-excludes it here too.
+ Provider-backed models (claude, gpt, linkup, nano-banana, ...) are intentionally NOT excluded: the
+ gateway genuinely proxies those, so they belong in the faithful gateway cache.
+ """
+ internal_spec_path = Path(str(get_kit_configs_dir())) / "inference" / "backends" / f"{PipelexBackend.INTERNAL}.toml"
+ raw_specs = load_toml_from_path(path=str(internal_spec_path))
+ return {name for name, spec in raw_specs.items() if name != "defaults" and isinstance(spec, dict)}
+
+
+def gateway_backend_model_specs_for_kit_deck() -> dict[str, Any]:
+ """Build a gateway ``backend_model_specs`` payload covering every concrete model handle the kit deck names.
+
+ Derived from the shipped kit deck rather than a hand-maintained list, so the primed
+ offline cache auto-tracks deck changes — e.g. promoting a premium alias to a new model
+ no longer silently breaks these tests. Every bare handle named in the deck's aliases,
+ waterfalls, and presets (across all model types) gets a minimal spec declaring the
+ gateway sdk for its type — all ``ModelManager._enforce_gateway_model_membership`` needs
+ is name membership. Deck/gateway consistency itself is covered by ``TestModelDeckReferences``.
+
+ Handles served by the software-only ``internal`` backend (see ``software_only_internal_handles``)
+ are excluded: the gateway never provides them, so claiming it does would make the fake cache
+ unfaithful and let a software-only extractor alias resolve through the gateway worker. Those
+ handles are still reachable in the offline subprocess via the local internal backend, so the
+ membership check passes for them without a gateway spec.
+ """
+ blueprint = load_kit_model_deck_blueprint()
+ excluded_handles = software_only_internal_handles()
+
+ handle_model_types: dict[str, ModelType] = {}
+ for model_type, section in (
+ (ModelType.LLM, blueprint.llm),
+ (ModelType.TEXT_EXTRACTOR, blueprint.extract),
+ (ModelType.IMG_GEN, blueprint.img_gen),
+ (ModelType.SEARCH, blueprint.search),
+ ):
+ for raw_reference in _deck_section_handle_references(section):
+ try:
+ ref = ModelReference.parse(raw_reference)
+ except ModelReferenceParseError:
+ continue
+ match ref.kind:
+ case ModelReferenceKind.HANDLE:
+ if ref.name in excluded_handles:
+ continue
+ handle_model_types.setdefault(ref.name, model_type)
+ case ModelReferenceKind.ALIAS | ModelReferenceKind.WATERFALL | ModelReferenceKind.PRESET:
+ continue
+
+ specs: dict[str, Any] = {"defaults": {"sdk": "gateway_completions"}}
+ for handle, model_type in handle_model_types.items():
+ specs[handle] = dict(_GATEWAY_SPEC_TEMPLATE_BY_TYPE[model_type])
+ return specs
+
def _copy_kit_configs_into(pipelex_dir: Path) -> None:
"""Mirror the kit's ``configs/`` tree into ``pipelex_dir`` so the subprocess can boot.
@@ -196,7 +314,10 @@ __all__ = [
"OFFLINE_BUNDLES_DIR",
"PIPELEX_AGENT_BIN",
"UNREACHABLE_REMOTE_CONFIG_URL",
+ "gateway_backend_model_specs_for_kit_deck",
+ "load_kit_model_deck_blueprint",
"set_gateway_enabled",
+ "software_only_internal_handles",
"write_active_routing_profile",
"write_remote_config_cache",
]
diff --git a/tests/e2e/agent_cli/test_offline_run_dry.py b/tests/e2e/agent_cli/test_offline_run_dry.py
index a9080bfaf..5f4597934 100644
--- a/tests/e2e/agent_cli/test_offline_run_dry.py
+++ b/tests/e2e/agent_cli/test_offline_run_dry.py
@@ -19,6 +19,8 @@ import pytest
from tests.e2e.agent_cli.conftest import (
OFFLINE_BUNDLES_DIR,
PIPELEX_AGENT_BIN,
+ gateway_backend_model_specs_for_kit_deck,
+ load_kit_model_deck_blueprint,
set_gateway_enabled,
write_active_routing_profile,
write_remote_config_cache,
@@ -104,81 +106,14 @@ def _parse_agent_json(output: str) -> dict[str, object]:
raise AssertionError(msg)
-# A minimal but structurally complete cached remote-config payload. ``backend_model_specs``
-# must include every terminal handle referenced by the default kit deck so the gateway
-# membership check passes for the "known-model" scenario.
-def _comprehensive_cached_backend_model_specs() -> dict[str, object]:
- """Build a ``backend_model_specs`` payload that satisfies the default kit deck.
+def _cached_remote_config_payload() -> dict[str, object]:
+ """A structurally complete cached remote-config payload for the primed-cache scenarios.
- The kit's ``1_llm_deck.toml`` references many concrete handles via aliases/waterfalls.
- We list them all here so the gateway-membership check in ``Pipelex.setup`` accepts the
- cache. The spec body uses ``gateway_completions`` (LLMs) / ``gateway_image`` (image-gen)
- sdks since the gateway proxies to those — values are minimal but valid against
- ``InferenceModelSpecBlueprint``.
+ ``backend_model_specs`` is derived from the shipped kit deck (see
+ ``gateway_backend_model_specs_for_kit_deck``) so the gateway-membership check in
+ ``Pipelex.setup`` accepts the cache without a hand-maintained handle list that goes
+ stale whenever the deck promotes an alias to a new model.
"""
- llm_handle_template: dict[str, object] = {
- "sdk": "gateway_completions",
- "model_type": "llm",
- "inputs": ["text"],
- "outputs": ["text", "structured"],
- }
- img_gen_handle_template: dict[str, object] = {
- "sdk": "gateway_image",
- "model_type": "img_gen",
- }
- extract_handle_template: dict[str, object] = {
- "sdk": "gateway_extract",
- "model_type": "text_extractor",
- }
- search_handle_template: dict[str, object] = {
- "sdk": "gateway_search",
- "model_type": "search",
- }
-
- llm_handles = [
- "claude-3.7-sonnet",
- "claude-4.5-haiku",
- "claude-4.5-sonnet",
- "claude-4.6-opus",
- "claude-4.6-sonnet",
- "claude-4.7-opus",
- "claude-4.8-opus",
- "gemini-flash-latest",
- "gemini-flash-lite-latest",
- "gemini-pro-latest",
- "gpt-4o",
- "gpt-4o-mini",
- "gpt-5",
- "gpt-5-mini",
- "gpt-5-nano",
- "gpt-5.1",
- "gpt-5.2",
- "mistral-large",
- ]
- img_gen_handles = [
- "gpt-image-1",
- "gpt-image-1-mini",
- "gpt-image-2",
- "nano-banana",
- "nano-banana-2",
- "nano-banana-pro",
- ]
- extract_handles = ["azure-document-intelligence", "linkup-fetch"]
- search_handles = ["linkup-standard", "linkup-deep"]
-
- specs: dict[str, object] = {"defaults": {"sdk": "gateway_completions"}}
- for handle in llm_handles:
- specs[handle] = dict(llm_handle_template)
- for handle in img_gen_handles:
- specs[handle] = dict(img_gen_handle_template)
- for handle in extract_handles:
- specs[handle] = dict(extract_handle_template)
- for handle in search_handles:
- specs[handle] = dict(search_handle_template)
- return specs
-
-
-def _cached_remote_config_payload() -> dict[str, object]:
return {
"posthog": {
"project_api_key": "test-key",
@@ -186,7 +121,7 @@ def _cached_remote_config_payload() -> dict[str, object]:
"is_geoip_enabled": False,
"is_debug_enabled": False,
},
- "backend_model_specs": _comprehensive_cached_backend_model_specs(),
+ "backend_model_specs": gateway_backend_model_specs_for_kit_deck(),
"aws_region": "us-east-1",
}
@@ -253,6 +188,44 @@ class TestOfflineDryRun:
message = str(payload.get("message", ""))
assert "pipelex init" in message.lower(), f"Error must mention `pipelex init` remediation; got: {message!r}"
+ def test_primed_cache_specs_track_kit_deck_premium_alias(self) -> None:
+ """Regression: the primed gateway cache is derived from the kit deck, so it tracks alias promotions.
+
+ Reads the deck's own ``default-premium`` target — whatever model it currently points
+ to — and asserts the derived gateway specs cover it, without hardcoding the model name.
+ If a future deck promotes the premium alias to a new handle this keeps passing; if the
+ derivation ever stops covering a referenced handle it fails fast here (a unit-speed
+ check) instead of deep inside a subprocess setup with an opaque GatewayUnknownModelError.
+ """
+ blueprint = load_kit_model_deck_blueprint()
+ premium_target = blueprint.llm.aliases["default-premium"]
+ specs = gateway_backend_model_specs_for_kit_deck()
+ assert premium_target in specs, (
+ f"Derived gateway specs must cover the kit deck's default-premium target '{premium_target}'; got handles: {sorted(specs)}"
+ )
+
+ def test_primed_cache_excludes_software_only_internal_handles(self) -> None:
+ """Regression: the gateway cache must not claim software-only internal extractors as gateway models.
+
+ ``@default-no-inference`` / ``@default-text-from-pdf`` resolve to an extractor served by the
+ local ``internal`` backend (``PipelexBackend.INTERNAL`` — "runs internally, without AI"), which
+ the Pipelex Gateway never provides. If the derived gateway specs claimed it, an offline dry-run
+ could resolve the alias to the fake ``gateway_extract`` worker instead of exercising the real
+ internal backend. The companion assertion guards the other direction: a genuinely gateway-served
+ extract model (the ``default-extract-document`` target) must stay covered, so the exclusion does
+ not over-reach onto provider-backed models the gateway does proxy.
+ """
+ blueprint = load_kit_model_deck_blueprint()
+ software_only_target = blueprint.extract.aliases["default-no-inference"]
+ gateway_extract_target = blueprint.extract.aliases["default-extract-document"]
+ specs = gateway_backend_model_specs_for_kit_deck()
+ assert software_only_target not in specs, (
+ f"Derived gateway specs must exclude software-only internal handle '{software_only_target}'; got handles: {sorted(specs)}"
+ )
+ assert gateway_extract_target in specs, (
+ f"Gateway-served extract target '{gateway_extract_target}' must remain covered; got handles: {sorted(specs)}"
+ )
+
def test_gateway_known_with_cache_succeeds_offline(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:
"""Gateway enabled + no network + primed cache → dry-run exits 0 with stale-cache warning."""
pipelex_dir = hermetic_home / ".pipelex"
@@ -270,6 +243,32 @@ class TestOfflineDryRun:
f"Cached-source setup must surface a RemoteConfigStale warning on the envelope; got: {payload!r}"
)
+ def test_gateway_img_gen_with_cache_succeeds_offline(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:
+ """Gateway enabled + no network + primed cache → an image-gen pipe dry-runs offline.
+
+ Companion to ``test_gateway_known_with_cache_succeeds_offline``, but the bundle's pipe is a
+ ``PipeImgGen`` referencing a gateway image handle (``nano-banana``). It proves the derived
+ cache covers img_gen handles too — not just LLMs — so the gateway-membership check passes for
+ an image model and the offline dry-run completes.
+
+ Scope note: dry-run mocks image generation via ``ContentGeneratorDry``, so this does NOT build
+ the img-gen worker and therefore does not exercise the spec's ``sdk`` -> worker-factory mapping;
+ it covers membership + handle resolution for image models. The sdk-vs-factory contract is a
+ runtime (non-dry) concern.
+ """
+ pipelex_dir = hermetic_home / ".pipelex"
+ set_gateway_enabled(pipelex_dir / "inference" / "backends.toml", enabled=True)
+ write_remote_config_cache(pipelex_dir, _cached_remote_config_payload())
+
+ staged_bundle = _stage_bundle(OFFLINE_BUNDLES_DIR / "gateway_img_gen_model", hermetic_home)
+ result = _run_agent_bundle(staged_bundle, offline_subprocess_env, cwd=hermetic_home)
+
+ assert result.returncode == 0, (
+ f"Gateway img-gen dry-run with primed cache must succeed offline.\nstdout={result.stdout!r}\nstderr={result.stderr!r}"
+ )
+ payload = _parse_agent_json(result.stdout)
+ assert "error" not in payload, payload
+
def test_gateway_unknown_with_cache_fails_with_clear_error(self, hermetic_home: Path, offline_subprocess_env: dict[str, str]) -> None:
"""Bundle pipe references a model absent from the cached gateway specs → clear error.
diff --git a/tests/e2e/data/offline_mode/gateway_img_gen_model/gateway_img_gen_model.mthds b/tests/e2e/data/offline_mode/gateway_img_gen_model/gateway_img_gen_model.mthds
new file mode 100644
index 000000000..0b5a76720
--- /dev/null
+++ b/tests/e2e/data/offline_mode/gateway_img_gen_model/gateway_img_gen_model.mthds
@@ -0,0 +1,11 @@
+domain = "offline_gateway_img_gen"
+description = "Pipeline referencing a gateway image-generation model that exists in the cached gateway specs"
+main_pipe = "gateway_img_gen_pipe"
+
+[pipe]
+[pipe.gateway_img_gen_pipe]
+type = "PipeImgGen"
+description = "Single image-generation step using a gateway-listed image model handle"
+output = "Image"
+model = "nano-banana"
+prompt = "A single red circle on a white background."