Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Rename search presets to provider-agnostic names (#737)

Louis Choquel7c128dec82c2
diff --git a/.pipelex-dev/test_profiles.toml b/.pipelex-dev/test_profiles.toml
index fc185277c..7b1588627 100644
--- a/.pipelex-dev/test_profiles.toml
+++ b/.pipelex-dev/test_profiles.toml
@@ -206,6 +206,10 @@ google = ["nano-banana", "nano-banana-pro"]
# --- Qwen Models ---
qwen = ["qwen-image"]
+[collections.search]
+# --- Search Models ---
+linkup = ["linkup-standard", "linkup-deep"]
+
[collections.extract]
# --- PDF Extraction Models ---
from_pdf = [
@@ -243,6 +247,7 @@ backends = ["azure_openai"]
llm_models = ["gpt-4o-mini"]
img_gen_models = []
extract_models = []
+search_models = []
################################################################################
# Dev Profile
@@ -253,6 +258,7 @@ backends = ["pipelex_gateway", "internal"]
llm_models = ["claude-4.5-haiku", "gpt-4o-mini", "gemini-2.5-flash-lite"]
img_gen_models = ["gpt-image-1-mini"]
extract_models = ["@from_pdf"]
+search_models = ["linkup-standard"]
################################################################################
# Coverage Profile
@@ -268,6 +274,7 @@ llm_models = [
]
img_gen_models = ["gpt-image-1", "nano-banana"]
extract_models = ["pypdfium2-extract-pdf"]
+search_models = ["linkup-standard"]
################################################################################
# Claude Latest Profile
@@ -278,6 +285,7 @@ backends = ["anthropic", "pipelex_gateway"]
llm_models = ["claude-4.5-*"]
img_gen_models = []
extract_models = []
+search_models = []
################################################################################
# GPT No Codex Profile
@@ -288,6 +296,7 @@ backends = ["openai", "pipelex_gateway"]
llm_models = ["gpt-*", "!*-codex", "!*-codex-*"]
img_gen_models = ["gpt-image-*"]
extract_models = []
+search_models = []
################################################################################
# All Backends Profile - Test across all backends with select models
@@ -298,6 +307,7 @@ backends = ["@all"]
llm_models = ["claude-4.5-haiku", "gpt-4o-mini", "gemini-2.5-flash-lite"]
img_gen_models = ["gpt-image-1-mini"]
extract_models = ["@from_pdf"]
+search_models = ["@linkup"]
################################################################################
# Full Profile - All available models (use with caution)
diff --git a/.pipelex/inference/backends/linkup.toml b/.pipelex/inference/backends/linkup.toml
index 9ff7d159f..184e324e9 100644
--- a/.pipelex/inference/backends/linkup.toml
+++ b/.pipelex/inference/backends/linkup.toml
@@ -24,14 +24,14 @@ thinking_mode = "none"
# SEARCH MODELS
################################################################################
-["linkup/standard"]
+["linkup-standard"]
model_id = "standard"
inputs = ["text"]
-outputs = ["structured"]
-costs = {}
+outputs = ["sourced-answers", "structured"]
+costs = { input = 0.005, output = 0 }
-["linkup/deep"]
+["linkup-deep"]
model_id = "deep"
inputs = ["text"]
-outputs = ["structured"]
-costs = {}
+outputs = ["sourced-answers", "structured"]
+costs = { input = 0.05, output = 0 }
diff --git a/.pipelex/inference/backends/pipelex_gateway_models.md b/.pipelex/inference/backends/pipelex_gateway_models.md
index 28b91b15a..bc4605314 100644
--- a/.pipelex/inference/backends/pipelex_gateway_models.md
+++ b/.pipelex/inference/backends/pipelex_gateway_models.md
@@ -446,13 +446,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
</tr>
-<tr>
-<td>mistral-document-ai-2505</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
-</tr>
</tbody>
</table>
@@ -522,6 +515,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T23:49:25Z
+> Last updated: 2026-03-04T00:21:08Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/.pipelex/inference/backends/pipelex_gateway_models_plain.md b/.pipelex/inference/backends/pipelex_gateway_models_plain.md
index d39ef8e6b..742c83039 100644
--- a/.pipelex/inference/backends/pipelex_gateway_models_plain.md
+++ b/.pipelex/inference/backends/pipelex_gateway_models_plain.md
@@ -163,9 +163,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **deepseek-ocr**
- inputs: image
- outputs: pages
-- **mistral-document-ai-2505**
- - inputs: image, pdf
- - outputs: pages
**About extracted pages:** Each page contains Markdown text (based on AI-interpreted layout) and optional extracted images. A single image input is treated as one page. Pipelex also wraps the `pypdfium2` library for raw text (without any AI interpretation) and images extraction and page views rendering. All these elements can be used as inputs into downstream pipes, including LLM prompts.
@@ -196,6 +193,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T23:49:25Z
+> Last updated: 2026-03-04T00:21:08Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/.pipelex/inference/deck/4_search_deck.toml b/.pipelex/inference/deck/4_search_deck.toml
index 9c5433936..6cc955d7e 100644
--- a/.pipelex/inference/deck/4_search_deck.toml
+++ b/.pipelex/inference/deck/4_search_deck.toml
@@ -19,7 +19,6 @@
####################################################################################################
[search]
-default_depth = "standard"
choice_default = "@default-search"
####################################################################################################
@@ -27,12 +26,12 @@ choice_default = "@default-search"
####################################################################################################
[search.aliases]
-default-search = "linkup/standard"
+default-search = "linkup-standard"
####################################################################################################
# Search Presets
####################################################################################################
[search.presets]
-linkup-standard = { model = "linkup/standard", depth = "standard", include_images = false, include_inline_citations = true }
-linkup-deep = { model = "linkup/deep", depth = "deep", include_images = false, include_inline_citations = true }
+standard = { model = "linkup-standard", include_images = false, include_inline_citations = true }
+deep = { model = "linkup-deep", include_images = false, include_inline_citations = true }
diff --git a/.pipelex/inference/routing_profiles.toml b/.pipelex/inference/routing_profiles.toml
index 612338ae7..090d2d4be 100644
--- a/.pipelex/inference/routing_profiles.toml
+++ b/.pipelex/inference/routing_profiles.toml
@@ -27,8 +27,6 @@ active = "all_pipelex_gateway"
[profiles.all_pipelex_gateway]
description = "Use Pipelex Gateway for all its supported models"
default = "pipelex_gateway"
-[profiles.all_pipelex_gateway.routes]
-"linkup/*" = "linkup"
[profiles.all_anthropic]
description = "Use Anthropic backend for all its supported models"
@@ -62,6 +60,10 @@ default = "groq"
description = "Use HuggingFace backend for all its supported models"
default = "huggingface"
+[profiles.all_linkup]
+description = "Use Linkup backend for all its supported models"
+default = "linkup"
+
[profiles.all_mistral]
description = "Use Mistral backend for all its supported models"
default = "mistral"
diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml
index 6b594bceb..0fa16c4ac 100644
--- a/.pipelex/pipelex.toml
+++ b/.pipelex/pipelex.toml
@@ -59,6 +59,10 @@ ranksep = 30 # Vertical spacing between ranks/levels
initial_zoom = 1.0 # Initial zoom level (1.0 = 100%)
pan_to_top = true # Pan to show top of graph on load
+[pipelex.pipeline_execution_config.graph_config.reactflow_config.style]
+theme = "dark"
+palette = "dracula"
+
####################################################################################################
# Storage Config
####################################################################################################
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e031e8fc8..804e99d95 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,7 @@
## [v0.19.0] - 2026-03-02
### Added
- - **Web Search Integration**: Introduced a new `PipeSearch` operator, native support for Linkup as a search backend provider, `SearchResult` and `SearchResultContent` concepts for handling answers with citations, and Model Deck support for search models with presets (e.g., `$linkup-standard`, `$linkup-deep`), aliases, and waterfalls.
+ - **Web Search Integration**: Introduced a new `PipeSearch` operator, native support for Linkup as a search backend provider, `SearchResult` and `SearchResultContent` concepts for handling answers with citations, and Model Deck support for search models.
- **Graph View Generation**: Added a `--view` option to `pipelex validate bundle` that generates a `ViewSpec` JSON (compatible with ReactFlow) for client-side graph rendering without writing files to disk.
### Changed
diff --git a/Makefile b/Makefile
index c0efe06e8..a98f10331 100644
--- a/Makefile
+++ b/Makefile
@@ -171,8 +171,8 @@ export HELP
merge-check-ruff-lint merge-check-ruff-format merge-check-mypy merge-check-pyright merge-check-plxt-format merge-check-plxt-lint \
li check-unused-imports fix-unused-imports check-TODOs check-uv \
docs docs-check docs-serve-versioned docs-list docs-deploy docs-deploy-stable docs-deploy-specific-version docs-delete \
- generate-mthds-schema gms check-mthds-schema cms \
- update-gateway-models ugm check-gateway-models cgm up \
+ generate-mthds-schema generate-mthds-schema-quiet gms check-mthds-schema cms \
+ update-gateway-models update-gateway-models-quiet ugm check-gateway-models cgm up \
test-count check-test-badge \
serve-graph serve-graph-bg stop-graph-server view-graph sg vg \
docs-deploy-root
@@ -300,6 +300,9 @@ generate-mthds-schema: env
$(call PRINT_TITLE,"Generating MTHDS JSON Schema")
$(VENV_PIPELEX_DEV) generate-mthds-schema
+generate-mthds-schema-quiet: env
+ $(VENV_PIPELEX_DEV) generate-mthds-schema --quiet
+
gms: generate-mthds-schema
@echo "> done: gms = generate-mthds-schema"
@@ -314,6 +317,9 @@ update-gateway-models: env
$(call PRINT_TITLE,"Updating gateway models reference")
$(VENV_PIPELEX_DEV) update-gateway-models
+update-gateway-models-quiet: env
+ $(VENV_PIPELEX_DEV) update-gateway-models --quiet
+
ugm: update-gateway-models
@echo "> done: ugm = update-gateway-models"
@@ -928,10 +934,10 @@ vg: view-graph
c: format lint pyright mypy
@echo "> done: c = check"
-cc: cleanderived regenerate-test-models-quiet generate-mthds-schema c
- @echo "> done: cc = cleanderived regenerate-test-models generate-mthds-schema format lint pyright pylint mypy"
+cc: cleanderived regenerate-test-models-quiet generate-mthds-schema-quiet update-gateway-models-quiet c
+ @echo "> done: cc = cleanderived regenerate-test-models generate-mthds-schema update-gateway-models format lint pyright mypy"
-up: generate-mthds-schema update-gateway-models up-kit-configs rules
+up: generate-mthds-schema-quiet update-gateway-models-quiet up-kit-configs rules
@echo "> done: up = generate-mthds-schema update-gateway-models up-kit-configs rules"
check: cc check-unused-imports check-config-sync check-rules check-urls check-gateway-models check-mthds-schema pylint
diff --git a/docs/home/6-build-reliable-ai-workflows/concepts/native-concepts.md b/docs/home/6-build-reliable-ai-workflows/concepts/native-concepts.md
index 9b2629c99..264255cb6 100644
--- a/docs/home/6-build-reliable-ai-workflows/concepts/native-concepts.md
+++ b/docs/home/6-build-reliable-ai-workflows/concepts/native-concepts.md
@@ -310,7 +310,7 @@ type = "PipeSearch"
description = "Search the web for information"
inputs = { topic = "Text" }
output = "SearchResult"
-model = "$linkup-standard"
+model = "$standard"
prompt = "What is $topic?"
```
diff --git a/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeSearch.md b/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeSearch.md
index e690e169f..5b16495f9 100644
--- a/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeSearch.md
+++ b/docs/home/6-build-reliable-ai-workflows/pipes/pipe-operators/PipeSearch.md
@@ -25,8 +25,8 @@ PipeSearch uses the unified inference backend system to manage search providers.
Common search presets:
-- `$linkup-standard`: Standard web search with fast results
-- `$linkup-deep`: Deep web search for more thorough results
+- `$standard`: Standard web search with fast results
+- `$deep`: Deep web search for more thorough results
### MTHDS Parameters
@@ -36,7 +36,7 @@ Common search presets:
| `description` | string | A description of the search operation. | Yes |
| `inputs` | dictionary | The input concept(s) for the search query, as a dictionary mapping input names to concept codes. Required when the prompt references variables. | No |
| `output` | string | The output concept produced by the search. Must be `SearchResult` or a concept that refines `SearchResult`. | Yes |
-| `model` | string | The search model preset to use (e.g., `"$linkup-standard"`, `"$linkup-deep"`). Defaults to the model specified in the global config. | No |
+| `model` | string | The search model preset to use (e.g., `"$standard"`, `"$deep"`). Defaults to the model specified in the global config. | No |
| `prompt` | string | The search query. Can be a static string or reference input variables using `$` prefix (e.g., `"What is $topic?"`). | Yes |
### Example: Static search query
@@ -48,7 +48,7 @@ This pipe performs a fixed search query without any inputs.
type = "PipeSearch"
description = "Search for the latest AI news"
output = "SearchResult"
-model = "$linkup-standard"
+model = "$standard"
prompt = "What are the latest developments in artificial intelligence?"
```
@@ -62,7 +62,7 @@ type = "PipeSearch"
description = "Search the web for information about a topic"
inputs = { topic = "Text" }
output = "SearchResult"
-model = "$linkup-standard"
+model = "$standard"
prompt = "What is $topic?"
```
@@ -76,7 +76,7 @@ type = "PipeSearch"
description = "Perform deep research on a topic"
inputs = { topic = "Text" }
output = "SearchResult"
-model = "$linkup-deep"
+model = "$deep"
prompt = "What are the main details about $topic?"
```
diff --git a/pipelex/builder/pipe/pipe_search_spec.py b/pipelex/builder/pipe/pipe_search_spec.py
index c69d73417..cc65ed109 100644
--- a/pipelex/builder/pipe/pipe_search_spec.py
+++ b/pipelex/builder/pipe/pipe_search_spec.py
@@ -80,7 +80,6 @@ class PipeSearchSpec(PipeSpec):
output=base_blueprint.output,
prompt=self.prompt,
model=search_choice,
- depth=None,
include_images=None,
max_results=None,
from_date=None,
diff --git a/pipelex/cli/commands/login/command.py b/pipelex/cli/commands/login/command.py
index 7c5d7481b..43b99facd 100644
--- a/pipelex/cli/commands/login/command.py
+++ b/pipelex/cli/commands/login/command.py
@@ -25,7 +25,7 @@ class CallbackHandler(BaseHTTPRequestHandler):
self._result = result
super().__init__(*args, **kwargs) # type: ignore[arg-type]
- def do_GET(self) -> None:
+ def do_GET(self) -> None: # pylint: disable=invalid-name
"""Handle GET /callback?api_key=xxx."""
parsed = urlparse(self.path)
params = parse_qs(parsed.query)
diff --git a/pipelex/cli/dev_cli/commands/preprocess_test_models_cmd.py b/pipelex/cli/dev_cli/commands/preprocess_test_models_cmd.py
index 1e18c9011..2b4c409a2 100644
--- a/pipelex/cli/dev_cli/commands/preprocess_test_models_cmd.py
+++ b/pipelex/cli/dev_cli/commands/preprocess_test_models_cmd.py
@@ -32,7 +32,7 @@ TEST_PROFILES_PATH = Path(ConfigPaths.DEV_CONFIG_DIR_PATH) / "test_profiles.toml
TEST_PROFILES_OVERRIDE_PATH = Path(ConfigPaths.DEV_CONFIG_DIR_PATH) / "test_profiles_override.toml"
# Model types we care about
-MODEL_TYPES = ["llm", "img_gen", "text_extractor"]
+MODEL_TYPES = ["llm", "img_gen", "text_extractor", "search"]
# Default model type from backend config
DEFAULT_MODEL_TYPE = "llm"
@@ -60,6 +60,7 @@ def _extract_models_from_backend_toml(backend_path: Path) -> dict[str, list[str]
"llm": [],
"img_gen": [],
"text_extractor": [],
+ "search": [],
}
for key, value in config.items():
@@ -97,7 +98,7 @@ def _fetch_gateway_models() -> dict[str, list[str]]:
remote_config = RemoteConfigFetcher.fetch_remote_config()
model_specs = dict(remote_config.backend_model_specs)
except (RemoteConfigFetchError, RemoteConfigValidationError):
- return {"llm": [], "img_gen": [], "text_extractor": []}
+ return {"llm": [], "img_gen": [], "text_extractor": [], "search": []}
# Get defaults
defaults = model_specs.get("defaults", {})
@@ -112,6 +113,7 @@ def _fetch_gateway_models() -> dict[str, list[str]]:
"llm": [],
"img_gen": [],
"text_extractor": [],
+ "search": [],
}
for key, value in model_specs.items():
@@ -156,6 +158,7 @@ def _collect_all_model_availability() -> dict[str, Any]:
"llm": {},
"img_gen": {},
"text_extractor": {},
+ "search": {},
}
backends_dir = Path(config_manager.backends_dir_path)
@@ -377,6 +380,7 @@ def _filter_models_by_profile(
"llm": [],
"img_gen": [],
"text_extractor": [],
+ "search": [],
}
# If include_all is set, return all valid pairs
@@ -426,6 +430,7 @@ def _filter_models_by_profile(
"llm_models": "llm",
"img_gen_models": "img_gen",
"extract_models": "text_extractor",
+ "search_models": "search",
}
# Model type to collection type mapping (internal name -> TOML section name)
@@ -433,6 +438,7 @@ def _filter_models_by_profile(
"llm": "llm",
"img_gen": "img_gen",
"text_extractor": "extract",
+ "search": "search",
}
# Resolve model lists from profile using advanced specifiers
@@ -529,6 +535,14 @@ def _generate_fixtures_python(
lines.append("]")
lines.append("")
+ # Search combos
+ search_pairs = combo_pairs.get("search", [])
+ lines.append("SEARCH_COMBOS: list[ModelCombo] = [")
+ for model, backend in sorted(search_pairs):
+ lines.append(f" ModelCombo({model!r}, {backend!r}),")
+ lines.append("]")
+ lines.append("")
+
return "\n".join(lines)
@@ -568,10 +582,12 @@ def _display_summary(
llm_total = sum(len(models) for models in availability.get("llm", {}).values())
img_gen_total = sum(len(models) for models in availability.get("img_gen", {}).values())
extract_total = sum(len(models) for models in availability.get("text_extractor", {}).values())
+ search_total = sum(len(models) for models in availability.get("search", {}).values())
table.add_row("LLM Models", str(llm_total), str(len(combo_pairs.get("llm", []))))
table.add_row("Image Gen Models", str(img_gen_total), str(len(combo_pairs.get("img_gen", []))))
table.add_row("Extract Models", str(extract_total), str(len(combo_pairs.get("text_extractor", []))))
+ table.add_row("Search Models", str(search_total), str(len(combo_pairs.get("search", []))))
table.add_row("", "", "")
table.add_row("Total Backends", str(len(total_backends)), "-")
table.add_row("Total Model/Backend Pairs", str(total_models), str(filtered_models))
diff --git a/pipelex/cogt/models/model_deck.py b/pipelex/cogt/models/model_deck.py
index cd411538b..08a4f3ce5 100644
--- a/pipelex/cogt/models/model_deck.py
+++ b/pipelex/cogt/models/model_deck.py
@@ -30,7 +30,6 @@ from pipelex.cogt.model_backends.constraints import ValuedConstraint
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.models.model_reference import ModelReference, ModelReferenceKind, ModelReferenceParseError, ensure_model_reference
-from pipelex.cogt.search.search_depth import SearchDepth
from pipelex.cogt.search.search_setting import SearchModelChoice, SearchSetting
from pipelex.system.configuration.config_model import ConfigModel
from pipelex.system.exceptions import ConfigValidationError
@@ -69,7 +68,6 @@ class ImgGenDeckBlueprint(ConfigModel):
class SearchDeckBlueprint(ConfigModel):
- default_depth: SearchDepth = Field(strict=False)
aliases: dict[str, str] = Field(default_factory=dict)
waterfalls: dict[str, list[str]] = Field(default_factory=dict)
presets: dict[str, SearchSetting] = Field(default_factory=dict)
@@ -115,7 +113,6 @@ class ModelDeck(ConfigModel):
img_gen_choice_default: ImgGenModelChoice
# Search-specific
- search_default_depth: SearchDepth = Field(strict=False)
search_aliases: dict[str, str] = Field(default_factory=dict)
search_waterfalls: dict[str, list[str]] = Field(default_factory=dict)
search_presets: dict[str, SearchSetting] = Field(default_factory=dict)
@@ -426,7 +423,7 @@ class ModelDeck(ConfigModel):
)
case ModelReferenceKind.ALIAS:
if alias_target := self.search_aliases.get(ref.name):
- return SearchSetting(model=alias_target, depth=self.search_default_depth)
+ return SearchSetting(model=alias_target)
msg = f"Alias '{ref.name}' was not found in the model deck"
raise ModelChoiceNotFoundError(
message=msg,
@@ -437,7 +434,7 @@ class ModelDeck(ConfigModel):
)
case ModelReferenceKind.WATERFALL:
if ref.name in self.search_waterfalls:
- return SearchSetting(model=ref.name, depth=self.search_default_depth)
+ return SearchSetting(model=ref.name)
msg = f"Waterfall '{ref.name}' was not found in the model deck"
raise ModelChoiceNotFoundError(
message=msg,
@@ -449,7 +446,7 @@ class ModelDeck(ConfigModel):
case ModelReferenceKind.HANDLE:
self._warn_if_ambiguous_search(ref.name)
if self.is_model_handle_defined(model_handle=ref.name, model_type=ModelType.SEARCH):
- return SearchSetting(model=ref.name, depth=self.search_default_depth)
+ return SearchSetting(model=ref.name)
self._raise_handle_not_found_error(
ref=ref,
model_type=ModelType.SEARCH,
diff --git a/pipelex/cogt/models/model_manager.py b/pipelex/cogt/models/model_manager.py
index fc3d4c9f1..1a636f78f 100644
--- a/pipelex/cogt/models/model_manager.py
+++ b/pipelex/cogt/models/model_manager.py
@@ -168,7 +168,6 @@ class ModelManager(ModelManagerAbstract):
img_gen_presets=model_deck_blueprint.img_gen.presets,
img_gen_choice_default=model_deck_blueprint.img_gen.choice_default,
# Search
- search_default_depth=model_deck_blueprint.search.default_depth,
search_aliases=model_deck_blueprint.search.aliases,
search_waterfalls=model_deck_blueprint.search.waterfalls,
search_presets=model_deck_blueprint.search.presets,
diff --git a/pipelex/cogt/search/fetch_job.py b/pipelex/cogt/search/fetch_job.py
new file mode 100644
index 000000000..1beac53b7
--- /dev/null
+++ b/pipelex/cogt/search/fetch_job.py
@@ -0,0 +1,35 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from typing_extensions import override
+
+from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.search.fetch_report import FetchTokensUsage
+
+
+class FetchJobReport(BaseModel):
+ fetch_tokens_usage: FetchTokensUsage | None = None
+
+
+class FetchJob(InferenceJobAbstract):
+ job_report: FetchJobReport = FetchJobReport()
+
+ @override
+ def validate_before_execution(self):
+ pass
+
+ def fetch_job_before_start(self, inference_model: InferenceModelSpec):
+ self.job_metadata.started_at = datetime.now()
+
+ self.job_report = FetchJobReport()
+ self.job_report.fetch_tokens_usage = FetchTokensUsage(
+ job_metadata=self.job_metadata,
+ inference_model_name=inference_model.name,
+ unit_costs=inference_model.costs,
+ inference_model_id=inference_model.model_id,
+ nb_tokens_by_category={},
+ )
+
+ def fetch_job_after_complete(self):
+ self.job_metadata.completed_at = datetime.now()
diff --git a/pipelex/cogt/search/fetch_report.py b/pipelex/cogt/search/fetch_report.py
new file mode 100644
index 000000000..0a68e8287
--- /dev/null
+++ b/pipelex/cogt/search/fetch_report.py
@@ -0,0 +1,72 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import StrEnum
+
+
+class FetchTokenCostReportField(StrEnum):
+ MODEL_TYPE = "model_type"
+ FETCH_NAME = "fetch_name"
+ PLATFORM_FETCH_ID = "platform_fetch_id"
+ NB_TOKENS_INPUT = "nb_tokens_input"
+ NB_TOKENS_INPUT_CACHED = "nb_tokens_input_cached"
+ NB_TOKENS_INPUT_NON_CACHED = "nb_tokens_input_non_cached"
+ NB_TOKENS_INPUT_JOINED = "nb_tokens_input_joined"
+ NB_TOKENS_OUTPUT = "nb_tokens_output"
+ COST_INPUT_CACHED = "cost_input_cached"
+ COST_INPUT_NON_CACHED = "cost_input_non_cached"
+ COST_INPUT_JOINED = "cost_input_joined"
+ COST_OUTPUT = "cost_output"
+
+ @staticmethod
+ def report_field_for_nb_tokens_by_category(token_category: TokenCategory) -> str:
+ return f"nb_tokens_{token_category}"
+
+ @staticmethod
+ def report_field_for_cost_by_category(token_category: CostCategory) -> str:
+ return f"cost_{token_category}"
+
+
+class FetchTokenCostReport(BaseModel):
+ model_type: str = "fetch"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ platform_model_id: str
+
+ nb_tokens_by_category: NbTokensByCategoryDict
+ costs_by_token_category: CostsByCategoryDict
+
+ def as_flat_dictionary(self) -> dict[str, Any]:
+ the_dict: dict[str, Any] = {}
+ dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)
+ the_dict.update(dict_for_job_metadata)
+ dict_for_model: dict[str, Any] = {
+ FetchTokenCostReportField.MODEL_TYPE: self.model_type,
+ FetchTokenCostReportField.FETCH_NAME: self.inference_model_name,
+ FetchTokenCostReportField.PLATFORM_FETCH_ID: self.platform_model_id,
+ }
+ the_dict.update(dict_for_model)
+ dict_for_nb_tokens = {
+ FetchTokenCostReportField.report_field_for_nb_tokens_by_category(token_category): nb_tokens
+ for token_category, nb_tokens in self.nb_tokens_by_category.items()
+ }
+ the_dict.update(dict_for_nb_tokens)
+ dict_for_costs = {
+ FetchTokenCostReportField.report_field_for_cost_by_category(token_category): cost
+ for token_category, cost in self.costs_by_token_category.items()
+ }
+ the_dict.update(dict_for_costs)
+ return the_dict
+
+
+class FetchTokensUsage(BaseModel):
+ model_type: str = "fetch"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ unit_costs: CostsByCategoryDict
+ inference_model_id: str
+ nb_tokens_by_category: NbTokensByCategoryDict
diff --git a/pipelex/cogt/search/fetch_worker_abstract.py b/pipelex/cogt/search/fetch_worker_abstract.py
new file mode 100644
index 000000000..f19e387d7
--- /dev/null
+++ b/pipelex/cogt/search/fetch_worker_abstract.py
@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+
+from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+
+
+class FetchWorkerAbstract(ABC):
+ def __init__(self, reporting_delegate: ReportingProtocol | None = None):
+ self.reporting_delegate = reporting_delegate
+
+ @abstractmethod
+ async def fetch_url(
+ self,
+ url: str,
+ job_metadata: JobMetadata,
+ include_raw_html: bool | None = None,
+ render_js: bool | None = None,
+ extract_images: bool | None = None,
+ timeout: float | None = None, # noqa: ASYNC109
+ ) -> TextAndImagesContent:
+ """Fetch and clean the content of a web page, returning text and optionally images.
+
+ Args:
+ url: The URL of the web page to fetch
+ job_metadata: Job metadata for cost reporting
+ include_raw_html: Whether to include the raw HTML of the webpage in the response
+ render_js: Whether to render the JavaScript of the webpage before fetching
+ extract_images: Whether to extract images from the webpage
+ timeout: The timeout for the HTTP request, in seconds
+
+ Returns:
+ TextAndImagesContent with the fetched page content
+ """
diff --git a/pipelex/cogt/search/search_job.py b/pipelex/cogt/search/search_job.py
new file mode 100644
index 000000000..7977987a5
--- /dev/null
+++ b/pipelex/cogt/search/search_job.py
@@ -0,0 +1,35 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+from typing_extensions import override
+
+from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.search.search_report import SearchTokensUsage
+
+
+class SearchJobReport(BaseModel):
+ search_tokens_usage: SearchTokensUsage | None = None
+
+
+class SearchJob(InferenceJobAbstract):
+ job_report: SearchJobReport = SearchJobReport()
+
+ @override
+ def validate_before_execution(self):
+ pass
+
+ def search_job_before_start(self, inference_model: InferenceModelSpec):
+ self.job_metadata.started_at = datetime.now()
+
+ self.job_report = SearchJobReport()
+ self.job_report.search_tokens_usage = SearchTokensUsage(
+ job_metadata=self.job_metadata,
+ inference_model_name=inference_model.name,
+ unit_costs=inference_model.costs,
+ inference_model_id=inference_model.model_id,
+ nb_tokens_by_category={},
+ )
+
+ def search_job_after_complete(self):
+ self.job_metadata.completed_at = datetime.now()
diff --git a/pipelex/cogt/search/search_job_factory.py b/pipelex/cogt/search/search_job_factory.py
new file mode 100644
index 000000000..6f5efeabe
--- /dev/null
+++ b/pipelex/cogt/search/search_job_factory.py
@@ -0,0 +1,17 @@
+from pipelex.cogt.search.fetch_job import FetchJob
+from pipelex.cogt.search.search_job import SearchJob
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
+
+
+class SearchJobFactory:
+ @classmethod
+ def make_search_job(cls, job_metadata: JobMetadata) -> SearchJob:
+ job_metadata.job_category = JobCategory.SEARCH_JOB
+ return SearchJob(job_metadata=job_metadata)
+
+
+class FetchJobFactory:
+ @classmethod
+ def make_fetch_job(cls, job_metadata: JobMetadata) -> FetchJob:
+ job_metadata.job_category = JobCategory.FETCH_JOB
+ return FetchJob(job_metadata=job_metadata)
diff --git a/pipelex/cogt/search/search_report.py b/pipelex/cogt/search/search_report.py
new file mode 100644
index 000000000..5b57784eb
--- /dev/null
+++ b/pipelex/cogt/search/search_report.py
@@ -0,0 +1,72 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import StrEnum
+
+
+class SearchTokenCostReportField(StrEnum):
+ MODEL_TYPE = "model_type"
+ SEARCH_NAME = "search_name"
+ PLATFORM_SEARCH_ID = "platform_search_id"
+ NB_TOKENS_INPUT = "nb_tokens_input"
+ NB_TOKENS_INPUT_CACHED = "nb_tokens_input_cached"
+ NB_TOKENS_INPUT_NON_CACHED = "nb_tokens_input_non_cached"
+ NB_TOKENS_INPUT_JOINED = "nb_tokens_input_joined"
+ NB_TOKENS_OUTPUT = "nb_tokens_output"
+ COST_INPUT_CACHED = "cost_input_cached"
+ COST_INPUT_NON_CACHED = "cost_input_non_cached"
+ COST_INPUT_JOINED = "cost_input_joined"
+ COST_OUTPUT = "cost_output"
+
+ @staticmethod
+ def report_field_for_nb_tokens_by_category(token_category: TokenCategory) -> str:
+ return f"nb_tokens_{token_category}"
+
+ @staticmethod
+ def report_field_for_cost_by_category(token_category: CostCategory) -> str:
+ return f"cost_{token_category}"
+
+
+class SearchTokenCostReport(BaseModel):
+ model_type: str = "search"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ platform_model_id: str
+
+ nb_tokens_by_category: NbTokensByCategoryDict
+ costs_by_token_category: CostsByCategoryDict
+
+ def as_flat_dictionary(self) -> dict[str, Any]:
+ the_dict: dict[str, Any] = {}
+ dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)
+ the_dict.update(dict_for_job_metadata)
+ dict_for_model: dict[str, Any] = {
+ SearchTokenCostReportField.MODEL_TYPE: self.model_type,
+ SearchTokenCostReportField.SEARCH_NAME: self.inference_model_name,
+ SearchTokenCostReportField.PLATFORM_SEARCH_ID: self.platform_model_id,
+ }
+ the_dict.update(dict_for_model)
+ dict_for_nb_tokens = {
+ SearchTokenCostReportField.report_field_for_nb_tokens_by_category(token_category): nb_tokens
+ for token_category, nb_tokens in self.nb_tokens_by_category.items()
+ }
+ the_dict.update(dict_for_nb_tokens)
+ dict_for_costs = {
+ SearchTokenCostReportField.report_field_for_cost_by_category(token_category): cost
+ for token_category, cost in self.costs_by_token_category.items()
+ }
+ the_dict.update(dict_for_costs)
+ return the_dict
+
+
+class SearchTokensUsage(BaseModel):
+ model_type: str = "search"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ unit_costs: CostsByCategoryDict
+ inference_model_id: str
+ nb_tokens_by_category: NbTokensByCategoryDict
diff --git a/pipelex/cogt/search/search_setting.py b/pipelex/cogt/search/search_setting.py
index 64acb78b8..a37eac148 100644
--- a/pipelex/cogt/search/search_setting.py
+++ b/pipelex/cogt/search/search_setting.py
@@ -3,20 +3,18 @@ from typing import Annotated, Union
from pydantic import BeforeValidator, Field
from pipelex.cogt.models.model_reference import ModelReference, parse_model_reference
-from pipelex.cogt.search.search_depth import SearchDepth
from pipelex.system.configuration.config_model import ConfigModel
class SearchSetting(ConfigModel):
model: str
- depth: SearchDepth = Field(strict=False)
include_images: bool = False
include_inline_citations: bool = True
max_results: int | None = Field(default=None, ge=1)
description: str | None = None
def desc(self) -> str:
- return f"SearchSetting(model={self.model}, depth={self.depth}, max_results={self.max_results})"
+ return f"SearchSetting(model={self.model}, max_results={self.max_results})"
# SearchModelChoice accepts SearchSetting, ModelReference, or a string (which gets parsed to ModelReference)
diff --git a/pipelex/cogt/search/search_worker_abstract.py b/pipelex/cogt/search/search_worker_abstract.py
index e6ae6a225..de70b2564 100644
--- a/pipelex/cogt/search/search_worker_abstract.py
+++ b/pipelex/cogt/search/search_worker_abstract.py
@@ -3,14 +3,20 @@ from typing import Any
from pipelex.cogt.search.search_setting import SearchSetting
from pipelex.core.stuffs.search_result_content import SearchResultContent
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.reporting.reporting_protocol import ReportingProtocol
class SearchWorkerAbstract(ABC):
+ def __init__(self, reporting_delegate: ReportingProtocol | None = None):
+ self.reporting_delegate = reporting_delegate
+
@abstractmethod
async def search_sourced_answer(
self,
query: str,
search_setting: SearchSetting,
+ job_metadata: JobMetadata,
include_domains: list[str] | None = None,
exclude_domains: list[str] | None = None,
from_date: str | None = None,
@@ -21,6 +27,7 @@ class SearchWorkerAbstract(ABC):
Args:
query: The search query text
search_setting: Search configuration including model, depth, etc.
+ job_metadata: Job metadata for cost reporting
include_domains: Optional list of domains to restrict search to
exclude_domains: Optional list of domains to exclude from search
from_date: Optional start date filter (YYYY-MM-DD)
@@ -36,6 +43,7 @@ class SearchWorkerAbstract(ABC):
query: str,
search_setting: SearchSetting,
output_schema: type,
+ job_metadata: JobMetadata,
include_domains: list[str] | None = None,
exclude_domains: list[str] | None = None,
from_date: str | None = None,
@@ -47,6 +55,7 @@ class SearchWorkerAbstract(ABC):
query: The search query text
search_setting: Search configuration including model, depth, etc.
output_schema: Pydantic model class defining the expected output structure
+ job_metadata: Job metadata for cost reporting
include_domains: Optional list of domains to restrict search to
exclude_domains: Optional list of domains to exclude from search
from_date: Optional start date filter (YYYY-MM-DD)
diff --git a/pipelex/cogt/search/search_worker_factory.py b/pipelex/cogt/search/search_worker_factory.py
index 985d8a2e8..811435198 100644
--- a/pipelex/cogt/search/search_worker_factory.py
+++ b/pipelex/cogt/search/search_worker_factory.py
@@ -1,35 +1,89 @@
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.search.fetch_worker_abstract import FetchWorkerAbstract
from pipelex.cogt.search.search_worker_abstract import SearchWorkerAbstract
+from pipelex.hub import get_models_manager, get_plugin_manager, get_report_delegate
+from pipelex.plugins.plugin import Plugin
-# Cache of search worker instances by provider prefix
-_search_workers: dict[str, SearchWorkerAbstract] = {}
+class SearchWorkerFactory:
+ @classmethod
+ def make_search_worker(
+ cls,
+ inference_model: InferenceModelSpec,
+ ) -> SearchWorkerAbstract:
+ """Create a search worker for the given inference model.
-def get_search_worker(model_handle: str) -> SearchWorkerAbstract:
- """Get a search worker instance for the given model handle.
+ Discriminates on plugin.sdk to select the appropriate implementation.
- The model handle format is "provider/variant" (e.g., "linkup/standard", "linkup/deep").
- The provider prefix determines which worker implementation to use.
+ Args:
+ inference_model: The model spec from the backend configuration.
- Args:
- model_handle: The search model handle (e.g., "linkup/standard")
+ Returns:
+ A SearchWorkerAbstract instance.
+ """
+ plugin = Plugin.make_for_inference_model(inference_model=inference_model)
+ backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)
+ plugin_sdk_registry = get_plugin_manager().plugin_sdk_registry
+ search_worker: SearchWorkerAbstract
+ match plugin.sdk:
+ case "linkup":
+ from pipelex.plugins.linkup.linkup_worker import LinkupWorker # noqa: PLC0415
- Returns:
- A SearchWorkerAbstract instance for the provider
- """
- provider = model_handle.split("/", maxsplit=1)[0] if "/" in model_handle else model_handle
+ search_worker = LinkupWorker(inference_model=inference_model, reporting_delegate=get_report_delegate())
+ case "gateway_search":
+ from pipelex.plugins.gateway.gateway_factory import GatewayFactory # noqa: PLC0415
+ from pipelex.plugins.gateway.gateway_search_worker import GatewaySearchWorker # noqa: PLC0415
- if provider in _search_workers:
- return _search_workers[provider]
+ sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GatewayFactory.make_portkey_client(backend=backend),
+ )
+ search_worker = GatewaySearchWorker(
+ sdk_instance=sdk_instance, inference_model=inference_model, reporting_delegate=get_report_delegate()
+ )
+ case _:
+ msg = f"Plugin '{plugin}' is not supported for search"
+ raise NotImplementedError(msg)
- worker: SearchWorkerAbstract
- match provider:
- case "linkup":
- from pipelex.plugins.linkup.linkup_search_worker import LinkupSearchWorker # noqa: PLC0415
+ return search_worker
- worker = LinkupSearchWorker()
- case _:
- msg = f"Unknown search provider: '{provider}' (from model handle '{model_handle}')"
- raise ValueError(msg)
+ @classmethod
+ def make_fetch_worker(
+ cls,
+ inference_model: InferenceModelSpec,
+ ) -> FetchWorkerAbstract:
+ """Create a fetch worker for the given inference model.
- _search_workers[provider] = worker
- return worker
+ Discriminates on plugin.sdk to select the appropriate implementation.
+
+ Args:
+ inference_model: The model spec from the backend configuration.
+
+ Returns:
+ A FetchWorkerAbstract instance.
+ """
+ plugin = Plugin.make_for_inference_model(inference_model=inference_model)
+ backend = get_models_manager().get_required_inference_backend(inference_model.backend_name)
+ plugin_sdk_registry = get_plugin_manager().plugin_sdk_registry
+ fetch_worker: FetchWorkerAbstract
+ match plugin.sdk:
+ case "linkup":
+ from pipelex.plugins.linkup.linkup_worker import LinkupWorker # noqa: PLC0415
+
+ fetch_worker = LinkupWorker(inference_model=inference_model, reporting_delegate=get_report_delegate())
+ case "gateway_search":
+ from pipelex.plugins.gateway.gateway_factory import GatewayFactory # noqa: PLC0415
+ from pipelex.plugins.gateway.gateway_fetch_worker import GatewayFetchWorker # noqa: PLC0415
+
+ sdk_instance = plugin_sdk_registry.get_sdk_instance(plugin=plugin) or plugin_sdk_registry.set_sdk_instance(
+ plugin=plugin,
+ sdk_instance=GatewayFactory.make_portkey_client(backend=backend),
+ )
+ fetch_worker = GatewayFetchWorker(
+ sdk_instance=sdk_instance, inference_model=inference_model, reporting_delegate=get_report_delegate()
+ )
+ case _:
+ msg = f"Plugin '{plugin}' is not supported for fetch"
+ raise NotImplementedError(msg)
+
+ return fetch_worker
diff --git a/pipelex/cogt/usage/cost_registry.py b/pipelex/cogt/usage/cost_registry.py
index 6d523ca9c..91b3abe43 100644
--- a/pipelex/cogt/usage/cost_registry.py
+++ b/pipelex/cogt/usage/cost_registry.py
@@ -12,14 +12,16 @@ from pipelex.cogt.exceptions import CostRegistryError
from pipelex.cogt.extract.extract_report import ExtractTokenCostReport, ExtractTokenCostReportField, ExtractTokensUsage
from pipelex.cogt.img_gen.img_gen_report import ImgGenTokenCostReport, ImgGenTokenCostReportField, ImgGenTokensUsage
from pipelex.cogt.llm.llm_report import LLMTokenCostReport, LLMTokenCostReportField, LLMTokensUsage
+from pipelex.cogt.search.fetch_report import FetchTokenCostReport, FetchTokenCostReportField, FetchTokensUsage
+from pipelex.cogt.search.search_report import SearchTokenCostReport, SearchTokenCostReportField, SearchTokensUsage
from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
from pipelex.cogt.usage.costs_per_token import model_cost_per_token
from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.hub import get_console
from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
-TokensUsage = LLMTokensUsage | ImgGenTokensUsage | ExtractTokensUsage
-TokenCostReport = LLMTokenCostReport | ImgGenTokenCostReport | ExtractTokenCostReport
+TokensUsage = LLMTokensUsage | ImgGenTokensUsage | ExtractTokensUsage | SearchTokensUsage | FetchTokensUsage
+TokenCostReport = LLMTokenCostReport | ImgGenTokenCostReport | ExtractTokenCostReport | SearchTokenCostReport | FetchTokenCostReport
CostRegistryRoot = list[TokenCostReport]
@@ -80,7 +82,9 @@ class CostRegistry(RootModel[CostRegistryRoot]):
model_name = (
record.get(report_field.LLM_NAME)
or record.get(ImgGenTokenCostReportField.IMG_GEN_NAME)
- or record.get(ExtractTokenCostReportField.EXTRACT_NAME, "unknown")
+ or record.get(ExtractTokenCostReportField.EXTRACT_NAME)
+ or record.get(SearchTokenCostReportField.SEARCH_NAME)
+ or record.get(FetchTokenCostReportField.FETCH_NAME, "unknown")
)
model_type = record.get(report_field.MODEL_TYPE, "llm")
model_types[model_name] = model_type
@@ -231,6 +235,24 @@ class CostRegistry(RootModel[CostRegistryRoot]):
nb_tokens_by_category=tokens_usage.nb_tokens_by_category,
costs_by_token_category=costs_by_token_category,
)
+ if isinstance(tokens_usage, SearchTokensUsage):
+ return SearchTokenCostReport(
+ model_type=tokens_usage.model_type,
+ job_metadata=tokens_usage.job_metadata,
+ inference_model_name=tokens_usage.inference_model_name,
+ platform_model_id=tokens_usage.inference_model_id,
+ nb_tokens_by_category=tokens_usage.nb_tokens_by_category,
+ costs_by_token_category=costs_by_token_category,
+ )
+ if isinstance(tokens_usage, FetchTokensUsage):
+ return FetchTokenCostReport(
+ model_type=tokens_usage.model_type,
+ job_metadata=tokens_usage.job_metadata,
+ inference_model_name=tokens_usage.inference_model_name,
+ platform_model_id=tokens_usage.inference_model_id,
+ nb_tokens_by_category=tokens_usage.nb_tokens_by_category,
+ costs_by_token_category=costs_by_token_category,
+ )
return ExtractTokenCostReport(
model_type=tokens_usage.model_type,
job_metadata=tokens_usage.job_metadata,
diff --git a/pipelex/core/stuffs/text_and_images_content.py b/pipelex/core/stuffs/text_and_images_content.py
index 6d78ba707..83fecdeff 100644
--- a/pipelex/core/stuffs/text_and_images_content.py
+++ b/pipelex/core/stuffs/text_and_images_content.py
@@ -16,6 +16,7 @@ from pipelex.tools.misc.pretty import PrettyPrintable
class TextAndImagesContent(StuffContent):
text: TextContent | None = Field(default=None, description="A text content")
images: list[ImageContent] | None = Field(default=None, description="A list of images that were extracted from the text")
+ raw_html: str | None = Field(default=None, description="The raw HTML of the fetched page, if requested")
@property
@override
@@ -43,6 +44,8 @@ class TextAndImagesContent(StuffContent):
# TODO: include the images into the HTML rendering
@override
def rendered_html(self) -> str:
+ if self.raw_html:
+ return self.raw_html
if self.text:
return self.text.rendered_html()
return ""
@@ -62,6 +65,8 @@ class TextAndImagesContent(StuffContent):
# TODO: include the images into the HTML rendering
@override
async def rendered_html_async(self) -> str:
+ if self.raw_html:
+ return self.raw_html
if self.text:
rendered = await self.text.rendered_html_async()
else:
diff --git a/pipelex/kit/configs/inference/backends/linkup.toml b/pipelex/kit/configs/inference/backends/linkup.toml
index 9ff7d159f..184e324e9 100644
--- a/pipelex/kit/configs/inference/backends/linkup.toml
+++ b/pipelex/kit/configs/inference/backends/linkup.toml
@@ -24,14 +24,14 @@ thinking_mode = "none"
# SEARCH MODELS
################################################################################
-["linkup/standard"]
+["linkup-standard"]
model_id = "standard"
inputs = ["text"]
-outputs = ["structured"]
-costs = {}
+outputs = ["sourced-answers", "structured"]
+costs = { input = 0.005, output = 0 }
-["linkup/deep"]
+["linkup-deep"]
model_id = "deep"
inputs = ["text"]
-outputs = ["structured"]
-costs = {}
+outputs = ["sourced-answers", "structured"]
+costs = { input = 0.05, output = 0 }
diff --git a/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md b/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
index 28b91b15a..8b0e2adc8 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
+++ b/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
@@ -446,13 +446,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
</tr>
-<tr>
-<td>mistral-document-ai-2505</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
-<td style="text-align:center;background-color:rgba(76,175,80,0.15)">❌</td>
-</tr>
</tbody>
</table>
@@ -522,6 +515,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T23:49:25Z
+> Last updated: 2026-03-03T22:21:10Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md b/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
index d39ef8e6b..4d38f25c9 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
+++ b/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
@@ -163,9 +163,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **deepseek-ocr**
- inputs: image
- outputs: pages
-- **mistral-document-ai-2505**
- - inputs: image, pdf
- - outputs: pages
**About extracted pages:** Each page contains Markdown text (based on AI-interpreted layout) and optional extracted images. A single image input is treated as one page. Pipelex also wraps the `pypdfium2` library for raw text (without any AI interpretation) and images extraction and page views rendering. All these elements can be used as inputs into downstream pipes, including LLM prompts.
@@ -196,6 +193,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T23:49:25Z
+> Last updated: 2026-03-03T22:21:10Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/pipelex/kit/configs/inference/deck/4_search_deck.toml b/pipelex/kit/configs/inference/deck/4_search_deck.toml
index 9c5433936..6cc955d7e 100644
--- a/pipelex/kit/configs/inference/deck/4_search_deck.toml
+++ b/pipelex/kit/configs/inference/deck/4_search_deck.toml
@@ -19,7 +19,6 @@
####################################################################################################
[search]
-default_depth = "standard"
choice_default = "@default-search"
####################################################################################################
@@ -27,12 +26,12 @@ choice_default = "@default-search"
####################################################################################################
[search.aliases]
-default-search = "linkup/standard"
+default-search = "linkup-standard"
####################################################################################################
# Search Presets
####################################################################################################
[search.presets]
-linkup-standard = { model = "linkup/standard", depth = "standard", include_images = false, include_inline_citations = true }
-linkup-deep = { model = "linkup/deep", depth = "deep", include_images = false, include_inline_citations = true }
+standard = { model = "linkup-standard", include_images = false, include_inline_citations = true }
+deep = { model = "linkup-deep", include_images = false, include_inline_citations = true }
diff --git a/pipelex/kit/configs/inference/routing_profiles.toml b/pipelex/kit/configs/inference/routing_profiles.toml
index 612338ae7..090d2d4be 100644
--- a/pipelex/kit/configs/inference/routing_profiles.toml
+++ b/pipelex/kit/configs/inference/routing_profiles.toml
@@ -27,8 +27,6 @@ active = "all_pipelex_gateway"
[profiles.all_pipelex_gateway]
description = "Use Pipelex Gateway for all its supported models"
default = "pipelex_gateway"
-[profiles.all_pipelex_gateway.routes]
-"linkup/*" = "linkup"
[profiles.all_anthropic]
description = "Use Anthropic backend for all its supported models"
@@ -62,6 +60,10 @@ default = "groq"
description = "Use HuggingFace backend for all its supported models"
default = "huggingface"
+[profiles.all_linkup]
+description = "Use Linkup backend for all its supported models"
+default = "linkup"
+
[profiles.all_mistral]
description = "Use Mistral backend for all its supported models"
default = "mistral"
diff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.toml
index 6b594bceb..0fa16c4ac 100644
--- a/pipelex/kit/configs/pipelex.toml
+++ b/pipelex/kit/configs/pipelex.toml
@@ -59,6 +59,10 @@ ranksep = 30 # Vertical spacing between ranks/levels
initial_zoom = 1.0 # Initial zoom level (1.0 = 100%)
pan_to_top = true # Pan to show top of graph on load
+[pipelex.pipeline_execution_config.graph_config.reactflow_config.style]
+theme = "dark"
+palette = "dracula"
+
####################################################################################################
# Storage Config
####################################################################################################
diff --git a/pipelex/pipe_operators/search/pipe_search.py b/pipelex/pipe_operators/search/pipe_search.py
index 6eba17f9f..0ff820734 100644
--- a/pipelex/pipe_operators/search/pipe_search.py
+++ b/pipelex/pipe_operators/search/pipe_search.py
@@ -7,9 +7,8 @@ from pipelex.cogt.content_generation.dry_run_factory import DryRunFactory
from pipelex.cogt.exceptions import ModelChoiceNotFoundError
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.models.model_deck_check import check_search_choice_with_deck
-from pipelex.cogt.search.search_depth import SearchDepth
from pipelex.cogt.search.search_setting import SearchModelChoice, SearchSetting
-from pipelex.cogt.search.search_worker_factory import get_search_worker
+from pipelex.cogt.search.search_worker_factory import SearchWorkerFactory
from pipelex.cogt.templating.template_blueprint import TemplateBlueprint
from pipelex.cogt.templating.template_rendering import render_template
from pipelex.core.memory.working_memory import WorkingMemory
@@ -35,7 +34,6 @@ class PipeSearch(PipeOperator[PipeSearchOutput]):
type: Literal["PipeSearch"] = "PipeSearch"
search_choice: SearchModelChoice | None
prompt_blueprint: TemplateBlueprint
- depth_override: SearchDepth | None = None
include_images_override: bool | None = None
max_results_override: int | None = None
from_date: str | None = None
@@ -105,15 +103,13 @@ class PipeSearch(PipeOperator[PipeSearchOutput]):
search_setting = search_setting.model_copy(update={"model": resolved_model_handle})
# 4. Apply pipe-level overrides
- if self.depth_override is not None:
- search_setting = search_setting.model_copy(update={"depth": self.depth_override})
if self.include_images_override is not None:
search_setting = search_setting.model_copy(update={"include_images": self.include_images_override})
if self.max_results_override is not None:
search_setting = search_setting.model_copy(update={"max_results": self.max_results_override})
# 5. Get search worker from factory
- worker = get_search_worker(model_handle=search_setting.model)
+ worker = SearchWorkerFactory.make_search_worker(inference_model=inference_model)
# 6. Execute search based on output type
content: StuffContent
@@ -121,6 +117,7 @@ class PipeSearch(PipeOperator[PipeSearchOutput]):
content = await worker.search_sourced_answer(
query=query_text,
search_setting=search_setting,
+ job_metadata=job_metadata,
include_domains=self.include_domains,
exclude_domains=self.exclude_domains,
from_date=self.from_date,
@@ -132,6 +129,7 @@ class PipeSearch(PipeOperator[PipeSearchOutput]):
query=query_text,
search_setting=search_setting,
output_schema=output_structure_class,
+ job_metadata=job_metadata,
include_domains=self.include_domains,
exclude_domains=self.exclude_domains,
from_date=self.from_date,
diff --git a/pipelex/pipe_operators/search/pipe_search_blueprint.py b/pipelex/pipe_operators/search/pipe_search_blueprint.py
index f38635e34..80cfae88d 100644
--- a/pipelex/pipe_operators/search/pipe_search_blueprint.py
+++ b/pipelex/pipe_operators/search/pipe_search_blueprint.py
@@ -2,7 +2,6 @@ from typing import Literal
from typing_extensions import override
-from pipelex.cogt.search.search_depth import SearchDepth
from pipelex.cogt.search.search_setting import SearchModelChoice
from pipelex.cogt.templating.template_category import TemplateCategory
from pipelex.cogt.templating.template_preprocessor import preprocess_template
@@ -18,7 +17,6 @@ class PipeSearchBlueprint(PipeBlueprint):
pipe_category: Literal["PipeOperator"] = "PipeOperator"
prompt: str
model: SearchModelChoice | None = None
- depth: SearchDepth | None = None
include_images: bool | None = None
max_results: int | None = None
from_date: str | None = None
diff --git a/pipelex/pipe_operators/search/pipe_search_factory.py b/pipelex/pipe_operators/search/pipe_search_factory.py
index c1428c402..18f262ff4 100644
--- a/pipelex/pipe_operators/search/pipe_search_factory.py
+++ b/pipelex/pipe_operators/search/pipe_search_factory.py
@@ -50,7 +50,6 @@ class PipeSearchFactory(PipeFactoryProtocol[PipeSearchBlueprint, PipeSearch]):
inputs=inputs,
search_choice=blueprint.model,
prompt_blueprint=prompt_blueprint,
- depth_override=blueprint.depth,
include_images_override=blueprint.include_images,
max_results_override=blueprint.max_results,
from_date=blueprint.from_date,
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index f15d4c47f..5fd13cb23 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -74,8 +74,8 @@ image-text-extractor = "@default-extract-image"
full-document-extractor = "@default-extract-document"
[pipelex.builder_config.talent_preset_mappings.search]
-web-search = "$linkup-standard"
-web-search-deep = "$linkup-deep"
+web-search = "$standard"
+web-search-deep = "$deep"
[pipelex.feature_config]
# WIP/Experimental feature flags
diff --git a/pipelex/pipeline/job_metadata.py b/pipelex/pipeline/job_metadata.py
index 9d03b178b..7631ae5cc 100644
--- a/pipelex/pipeline/job_metadata.py
+++ b/pipelex/pipeline/job_metadata.py
@@ -14,6 +14,8 @@ class JobCategory(StrEnum):
IMG_GEN_JOB = "img_gen_job"
JINJA2_JOB = "jinja2_job"
EXTRACT_JOB = "extract_job"
+ SEARCH_JOB = "search_job"
+ FETCH_JOB = "fetch_job"
class UnitJobId(StrEnum):
@@ -21,18 +23,23 @@ class UnitJobId(StrEnum):
LLM_GEN_OBJECT = "llm_gen_object"
IMG_GEN_TEXT_TO_IMAGE = "img_gen_text_to_image"
EXTRACT_PAGES = "extract_pages"
+ SEARCH_SOURCED_ANSWER = "search_sourced_answer"
+ SEARCH_STRUCTURED = "search_structured"
+ FETCH_URL = "fetch_url"
@property
def model_kind(self) -> str:
match self:
- case UnitJobId.LLM_GEN_TEXT:
- return "LLM"
- case UnitJobId.LLM_GEN_OBJECT:
+ case UnitJobId.LLM_GEN_TEXT | UnitJobId.LLM_GEN_OBJECT:
return "LLM"
case UnitJobId.IMG_GEN_TEXT_TO_IMAGE:
return "ImgGen"
case UnitJobId.EXTRACT_PAGES:
return "Extract"
+ case UnitJobId.SEARCH_SOURCED_ANSWER | UnitJobId.SEARCH_STRUCTURED:
+ return "Search"
+ case UnitJobId.FETCH_URL:
+ return "Fetch"
class JobMetadata(BaseModel):
diff --git a/pipelex/plugins/gateway/gateway_exceptions.py b/pipelex/plugins/gateway/gateway_exceptions.py
index 35eaac4b0..3228bf203 100644
--- a/pipelex/plugins/gateway/gateway_exceptions.py
+++ b/pipelex/plugins/gateway/gateway_exceptions.py
@@ -19,3 +19,7 @@ class GatewayDeckError(GatewayError):
class GatewayExtractResponseError(GatewayError):
pass
+
+
+class GatewaySearchResponseError(GatewayError):
+ pass
diff --git a/pipelex/plugins/gateway/gateway_fetch_worker.py b/pipelex/plugins/gateway/gateway_fetch_worker.py
new file mode 100644
index 000000000..a79b1f19b
--- /dev/null
+++ b/pipelex/plugins/gateway/gateway_fetch_worker.py
@@ -0,0 +1,179 @@
+import json
+from typing import Any, cast
+
+from portkey_ai import AsyncPortkey
+from portkey_ai.api_resources import exceptions as portkey_exceptions
+from portkey_ai.api_resources.utils import GenericResponse
+from tenacity import AsyncRetrying, RetryCallState, retry_if_exception, stop_after_attempt, wait_random_exponential
+from typing_extensions import override
+
+from pipelex import log
+from pipelex.cogt.exceptions import SdkTypeError
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.search.fetch_job import FetchJob
+from pipelex.cogt.search.fetch_worker_abstract import FetchWorkerAbstract
+from pipelex.cogt.search.search_job_factory import FetchJobFactory
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.config import get_config
+from pipelex.core.stuffs.image_content import ImageContent
+from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.pipeline.job_metadata import JobMetadata, UnitJobId
+from pipelex.plugins.gateway.gateway_deck import GatewayDeck
+from pipelex.plugins.gateway.gateway_exceptions import GatewaySearchResponseError
+from pipelex.plugins.gateway.gateway_factory import GatewayFactory
+from pipelex.plugins.gateway.gateway_search_schemas import GatewayFetchRequestParams
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+
+
+class GatewayFetchWorker(FetchWorkerAbstract):
+ """Fetch worker that routes through Portkey to the pipelex-relay LinkUp fetch endpoint."""
+
+ def __init__(
+ self,
+ sdk_instance: Any,
+ inference_model: InferenceModelSpec,
+ reporting_delegate: ReportingProtocol | None = None,
+ ):
+ super().__init__(reporting_delegate=reporting_delegate)
+
+ if not isinstance(sdk_instance, AsyncPortkey):
+ msg = f"Provided sdk_instance for {self.__class__.__name__} is not of type AsyncPortkey: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
+
+ self.portkey_client: AsyncPortkey = sdk_instance
+ self.inference_model = inference_model
+ self._tenacity_config = get_config().cogt.tenacity_config
+
+ def _make_retryer(self) -> AsyncRetrying:
+ """Create a fresh AsyncRetrying instance for each call."""
+ return AsyncRetrying(
+ retry=retry_if_exception(self._is_retryable_portkey_error),
+ before_sleep=self._log_retry,
+ wait=wait_random_exponential(
+ multiplier=self._tenacity_config.wait_multiplier,
+ max=self._tenacity_config.wait_max,
+ exp_base=self._tenacity_config.wait_exp_base,
+ ),
+ reraise=True,
+ stop=stop_after_attempt(self._tenacity_config.max_retries),
+ )
+
+ @override
+ async def fetch_url(
+ self,
+ url: str,
+ job_metadata: JobMetadata,
+ include_raw_html: bool | None = None,
+ render_js: bool | None = None,
+ extract_images: bool | None = None,
+ timeout: float | None = None, # noqa: ASYNC109
+ ) -> TextAndImagesContent:
+ job_metadata.unit_job_id = UnitJobId.FETCH_URL
+ fetch_job = FetchJobFactory.make_fetch_job(job_metadata=job_metadata)
+ fetch_job.fetch_job_before_start(inference_model=self.inference_model)
+
+ params = GatewayFetchRequestParams(
+ url=url,
+ include_raw_html=include_raw_html,
+ render_js=render_js,
+ extract_images=extract_images,
+ timeout=timeout,
+ )
+
+ config_id = GatewayDeck.get_config_id(headers=self.inference_model.extra_headers or {})
+ log.dev(f"Fetch via gateway config '{config_id}'")
+
+ messages: list[dict[str, str]] = [{"role": "user", "content": params.model_dump_json()}]
+
+ attempt_number = 0
+ response: GenericResponse | None = None
+ retryer = self._make_retryer()
+ try:
+ async for attempt in retryer:
+ with attempt:
+ attempt_number += 1
+ response = await self.portkey_client.with_options(config=config_id).post( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+ "/chat/completions",
+ model="linkup/fetch",
+ messages=messages,
+ )
+ except portkey_exceptions.APIError as exc:
+ error_summary = GatewayFactory.make_error_summary_from_portkey_error(exc)
+ msg = f"Fetch service error after {attempt_number} attempt(s): {error_summary}"
+ raise GatewaySearchResponseError(msg) from exc
+
+ if response is None:
+ msg = f"Could not get a fetch response via Portkey after {attempt_number} attempts"
+ raise GatewaySearchResponseError(msg)
+
+ if not isinstance(response, GenericResponse):
+ msg = "Response is not of type GenericResponse"
+ raise TypeError(msg)
+
+ self._extract_usage(response=response, fetch_job=fetch_job)
+ fetch_job.fetch_job_after_complete()
+ if self.reporting_delegate:
+ self.reporting_delegate.report_inference_job(inference_job=fetch_job)
+
+ # Extract content from response — use dict access (GenericResponse uses extra="allow")
+ try:
+ choice: dict[str, Any] = response.choices[0] # type: ignore[attr-defined] # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ raw_content = cast("object", choice["message"]["content"])
+ except (KeyError, IndexError, TypeError) as exc:
+ msg = "Could not extract content from gateway fetch response"
+ raise GatewaySearchResponseError(msg) from exc
+
+ if not isinstance(raw_content, str):
+ msg = f"Expected string content in fetch response, got {type(raw_content)}"
+ raise GatewaySearchResponseError(msg)
+ content_str: str = raw_content
+
+ result_dict: dict[str, Any] = json.loads(content_str)
+
+ text = TextContent(text=result_dict["markdown"]) if result_dict.get("markdown") else None
+
+ images: list[ImageContent] | None = None
+ if result_dict.get("images") is not None:
+ images = [
+ ImageContent(
+ url=image["url"],
+ caption=image.get("alt"),
+ )
+ for image in result_dict["images"]
+ ]
+
+ return TextAndImagesContent(
+ text=text,
+ images=images or None,
+ raw_html=result_dict.get("raw_html"),
+ )
+
+ def _extract_usage(self, response: GenericResponse, fetch_job: FetchJob) -> None:
+ """Extract token usage from the GenericResponse and populate the fetch job report."""
+ response_dict: dict[str, Any] = response.model_dump(serialize_as_any=True)
+ fetch_tokens_usage = fetch_job.job_report.fetch_tokens_usage
+ if (usage_dict := response_dict.get("usage")) and fetch_tokens_usage:
+ nb_tokens: NbTokensByCategoryDict = {}
+ if input_tokens := usage_dict.get("prompt_tokens") or usage_dict.get("input_tokens"):
+ nb_tokens[TokenCategory.INPUT] = input_tokens
+ if output_tokens := usage_dict.get("completion_tokens") or usage_dict.get("output_tokens"):
+ nb_tokens[TokenCategory.OUTPUT] = output_tokens
+ fetch_tokens_usage.nb_tokens_by_category = nb_tokens
+
+ def _is_retryable_portkey_error(self, exc: BaseException) -> bool:
+ if isinstance(exc, portkey_exceptions.NotFoundError):
+ msg = str(exc).lower()
+ return "specified deployment could not be found" in msg
+ return False
+
+ def _log_retry(self, retry_state: RetryCallState) -> None:
+ """Called before sleeping between retries."""
+ if not retry_state.outcome:
+ log.error("Tenacity retry state outcome is None")
+ return
+ exc = retry_state.outcome.exception()
+ attempt = retry_state.attempt_number
+ wait_duration = retry_state.next_action.sleep if retry_state.next_action else 0.0
+ log.dev(f"{self.__class__.__name__} retry #{attempt} for fetch due to '{type(exc).__name__}'.")
+ log.verbose(f"Wait duration before next attempt: {wait_duration:.4f}s")
diff --git a/pipelex/plugins/gateway/gateway_search_schemas.py b/pipelex/plugins/gateway/gateway_search_schemas.py
new file mode 100644
index 000000000..adec04b8c
--- /dev/null
+++ b/pipelex/plugins/gateway/gateway_search_schemas.py
@@ -0,0 +1,28 @@
+from pydantic import BaseModel, Field
+
+from pipelex.cogt.search.search_depth import SearchDepth
+
+
+class GatewaySearchRequestParams(BaseModel):
+ """Parameters for a web search request sent to the relay."""
+
+ query: str = Field(description="The search query text")
+ depth: SearchDepth = Field(default=SearchDepth.STANDARD, strict=False, description="Search depth: 'standard' or 'deep'")
+ include_images: bool = Field(default=False, description="Whether to include images in results")
+ include_inline_citations: bool = Field(default=True, description="Whether to include inline citations")
+ max_results: int | None = Field(default=None, description="Maximum number of results to return")
+ include_domains: list[str] | None = Field(default=None, description="Restrict search to these domains")
+ exclude_domains: list[str] | None = Field(default=None, description="Exclude these domains from search")
+ from_date: str | None = Field(default=None, description="Start date filter (YYYY-MM-DD)")
+ to_date: str | None = Field(default=None, description="End date filter (YYYY-MM-DD)")
+ output_schema: dict[str, object] | None = Field(default=None, description="JSON Schema for structured search output")
+
+
+class GatewayFetchRequestParams(BaseModel):
+ """Parameters for a web page fetch request sent to the relay."""
+
+ url: str = Field(description="The URL to fetch")
+ include_raw_html: bool | None = Field(default=None, description="Whether to include raw HTML in response")
+ render_js: bool | None = Field(default=None, description="Whether to render JavaScript before fetching")
+ extract_images: bool | None = Field(default=None, description="Whether to extract images from the page")
+ timeout: float | None = Field(default=None, description="Request timeout in seconds")
diff --git a/pipelex/plugins/gateway/gateway_search_worker.py b/pipelex/plugins/gateway/gateway_search_worker.py
new file mode 100644
index 000000000..e3399ef43
--- /dev/null
+++ b/pipelex/plugins/gateway/gateway_search_worker.py
@@ -0,0 +1,253 @@
+import json
+from typing import Any, cast
+
+from portkey_ai import AsyncPortkey
+from portkey_ai.api_resources import exceptions as portkey_exceptions
+from portkey_ai.api_resources.utils import GenericResponse
+from tenacity import AsyncRetrying, RetryCallState, retry_if_exception, stop_after_attempt, wait_random_exponential
+from typing_extensions import override
+
+from pipelex import log
+from pipelex.cogt.exceptions import SdkTypeError
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.search.search_depth import SearchDepth
+from pipelex.cogt.search.search_job import SearchJob
+from pipelex.cogt.search.search_job_factory import SearchJobFactory
+from pipelex.cogt.search.search_setting import SearchSetting
+from pipelex.cogt.search.search_worker_abstract import SearchWorkerAbstract
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.config import get_config
+from pipelex.core.stuffs.search_result_content import SearchResultContent, SearchSourceContent
+from pipelex.pipeline.job_metadata import JobMetadata, UnitJobId
+from pipelex.plugins.gateway.gateway_deck import GatewayDeck
+from pipelex.plugins.gateway.gateway_exceptions import GatewaySearchResponseError
+from pipelex.plugins.gateway.gateway_factory import GatewayFactory
+from pipelex.plugins.gateway.gateway_search_schemas import GatewaySearchRequestParams
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+
+
+class GatewaySearchWorker(SearchWorkerAbstract):
+ """Search worker that routes through Portkey to the pipelex-relay LinkUp endpoints."""
+
+ def __init__(
+ self,
+ sdk_instance: Any,
+ inference_model: InferenceModelSpec,
+ reporting_delegate: ReportingProtocol | None = None,
+ ):
+ super().__init__(reporting_delegate=reporting_delegate)
+
+ if not isinstance(sdk_instance, AsyncPortkey):
+ msg = f"Provided sdk_instance for {self.__class__.__name__} is not of type AsyncPortkey: it's a '{type(sdk_instance)}'"
+ raise SdkTypeError(msg)
+
+ self.portkey_client: AsyncPortkey = sdk_instance
+ self.inference_model = inference_model
+ self._tenacity_config = get_config().cogt.tenacity_config
+
+ def _make_retryer(self) -> AsyncRetrying:
+ """Create a fresh AsyncRetrying instance for each call."""
+ return AsyncRetrying(
+ retry=retry_if_exception(self._is_retryable_portkey_error),
+ before_sleep=self._log_retry,
+ wait=wait_random_exponential(
+ multiplier=self._tenacity_config.wait_multiplier,
+ max=self._tenacity_config.wait_max,
+ exp_base=self._tenacity_config.wait_exp_base,
+ ),
+ reraise=True,
+ stop=stop_after_attempt(self._tenacity_config.max_retries),
+ )
+
+ @override
+ async def search_sourced_answer(
+ self,
+ query: str,
+ search_setting: SearchSetting,
+ job_metadata: JobMetadata,
+ include_domains: list[str] | None = None,
+ exclude_domains: list[str] | None = None,
+ from_date: str | None = None,
+ to_date: str | None = None,
+ ) -> SearchResultContent:
+ job_metadata.unit_job_id = UnitJobId.SEARCH_SOURCED_ANSWER
+ search_job = SearchJobFactory.make_search_job(job_metadata=job_metadata)
+ search_job.search_job_before_start(inference_model=self.inference_model)
+
+ params = GatewaySearchRequestParams(
+ query=query,
+ depth=SearchDepth(self.inference_model.model_id),
+ include_images=search_setting.include_images,
+ include_inline_citations=search_setting.include_inline_citations,
+ max_results=search_setting.max_results,
+ include_domains=include_domains,
+ exclude_domains=exclude_domains,
+ from_date=from_date,
+ to_date=to_date,
+ )
+
+ response = await self._call_relay(
+ model=self.inference_model.model_id,
+ content=params.model_dump_json(),
+ )
+
+ self._extract_usage(response=response, search_job=search_job)
+ search_job.search_job_after_complete()
+ if self.reporting_delegate:
+ self.reporting_delegate.report_inference_job(inference_job=search_job)
+
+ content_str = self._extract_content(response)
+ result_dict: dict[str, Any] = json.loads(content_str)
+
+ sources = [
+ SearchSourceContent(
+ name=source["name"],
+ url=source["url"],
+ snippet=source["snippet"],
+ )
+ for source in result_dict.get("sources", [])
+ ]
+
+ return SearchResultContent(
+ answer=result_dict["answer"],
+ sources=sources,
+ )
+
+ @override
+ async def search_structured(
+ self,
+ query: str,
+ search_setting: SearchSetting,
+ output_schema: type,
+ job_metadata: JobMetadata,
+ include_domains: list[str] | None = None,
+ exclude_domains: list[str] | None = None,
+ from_date: str | None = None,
+ to_date: str | None = None,
+ ) -> dict[str, Any]:
+ job_metadata.unit_job_id = UnitJobId.SEARCH_STRUCTURED
+ search_job = SearchJobFactory.make_search_job(job_metadata=job_metadata)
+ search_job.search_job_before_start(inference_model=self.inference_model)
+
+ # Convert Pydantic model class to JSON Schema dict for the relay
+ schema_dict: dict[str, Any]
+ if hasattr(output_schema, "model_json_schema"):
+ schema_dict = cast("dict[str, Any]", output_schema.model_json_schema()) # pyright: ignore[reportUnknownMemberType]
+ else:
+ msg = f"output_schema must be a Pydantic model class with model_json_schema(), got {output_schema}"
+ raise SdkTypeError(msg)
+
+ params = GatewaySearchRequestParams(
+ query=query,
+ depth=SearchDepth(self.inference_model.model_id),
+ include_images=search_setting.include_images,
+ include_inline_citations=search_setting.include_inline_citations,
+ max_results=search_setting.max_results,
+ include_domains=include_domains,
+ exclude_domains=exclude_domains,
+ from_date=from_date,
+ to_date=to_date,
+ output_schema=schema_dict,
+ )
+
+ response = await self._call_relay(
+ model=self.inference_model.model_id,
+ content=params.model_dump_json(),
+ )
+
+ self._extract_usage(response=response, search_job=search_job)
+ search_job.search_job_after_complete()
+ if self.reporting_delegate:
+ self.reporting_delegate.report_inference_job(inference_job=search_job)
+
+ content_str = self._extract_content(response)
+ result: dict[str, Any] = json.loads(content_str)
+ return result
+
+ def _extract_usage(self, response: GenericResponse, search_job: SearchJob) -> None:
+ """Extract token usage from the GenericResponse and populate the search job report."""
+ response_dict: dict[str, Any] = response.model_dump(serialize_as_any=True)
+ search_tokens_usage = search_job.job_report.search_tokens_usage
+ if (usage_dict := response_dict.get("usage")) and search_tokens_usage:
+ nb_tokens: NbTokensByCategoryDict = {}
+ if input_tokens := usage_dict.get("prompt_tokens") or usage_dict.get("input_tokens"):
+ nb_tokens[TokenCategory.INPUT] = input_tokens
+ if output_tokens := usage_dict.get("completion_tokens") or usage_dict.get("output_tokens"):
+ nb_tokens[TokenCategory.OUTPUT] = output_tokens
+ search_tokens_usage.nb_tokens_by_category = nb_tokens
+
+ async def _call_relay(self, model: str, content: str) -> GenericResponse:
+ """Send a request through Portkey to the relay.
+
+ Args:
+ model: The relay model identifier (e.g., "linkup-sourced-answer")
+ content: JSON-encoded request parameters
+
+ Returns:
+ The Portkey GenericResponse
+ """
+ config_id = GatewayDeck.get_config_id(headers=self.inference_model.extra_headers or {})
+ log.dev(f"Search via gateway config '{config_id}' with model '{model}'")
+
+ messages: list[dict[str, str]] = [{"role": "user", "content": content}]
+
+ attempt_number = 0
+ response: GenericResponse | None = None
+ retryer = self._make_retryer()
+ try:
+ async for attempt in retryer:
+ with attempt:
+ attempt_number += 1
+ response = await self.portkey_client.with_options(config=config_id).post( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+ "/chat/completions",
+ model=model,
+ messages=messages,
+ )
+ except portkey_exceptions.APIError as exc:
+ error_summary = GatewayFactory.make_error_summary_from_portkey_error(exc)
+ msg = f"Search service error for model '{model}' after {attempt_number} attempt(s): {error_summary}"
+ raise GatewaySearchResponseError(msg) from exc
+
+ if response is None:
+ msg = f"Could not get a response for model '{model}' via Portkey after {attempt_number} attempts"
+ raise GatewaySearchResponseError(msg)
+
+ if not isinstance(response, GenericResponse):
+ msg = "Response is not of type GenericResponse"
+ raise TypeError(msg)
+
+ return response
+
+ def _extract_content(self, response: GenericResponse) -> str:
+ """Extract the content string from a GenericResponse.
+
+ GenericResponse uses Pydantic's extra="allow", so `choices` is a raw
+ list[dict] rather than a list of typed objects — use dict access.
+ """
+ try:
+ choice: dict[str, Any] = response.choices[0] # type: ignore[attr-defined] # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType]
+ content = cast("object", choice["message"]["content"])
+ if not isinstance(content, str):
+ msg = f"Expected string content in response, got {type(content)}"
+ raise GatewaySearchResponseError(msg)
+ return content
+ except (KeyError, IndexError, TypeError) as exc:
+ msg = "Could not extract content from gateway search response"
+ raise GatewaySearchResponseError(msg) from exc
+
+ def _is_retryable_portkey_error(self, exc: BaseException) -> bool:
+ if isinstance(exc, portkey_exceptions.NotFoundError):
+ msg = str(exc).lower()
+ return "specified deployment could not be found" in msg
+ return False
+
+ def _log_retry(self, retry_state: RetryCallState) -> None:
+ """Called before sleeping between retries."""
+ if not retry_state.outcome:
+ log.error("Tenacity retry state outcome is None")
+ return
+ exc = retry_state.outcome.exception()
+ attempt = retry_state.attempt_number
+ wait_duration = retry_state.next_action.sleep if retry_state.next_action else 0.0
+ log.dev(f"{self.__class__.__name__} retry #{attempt} for search due to '{type(exc).__name__}'.")
+ log.verbose(f"Wait duration before next attempt: {wait_duration:.4f}s")
diff --git a/pipelex/plugins/linkup/linkup_search_worker.py b/pipelex/plugins/linkup/linkup_search_worker.py
deleted file mode 100644
index ac038d594..000000000
--- a/pipelex/plugins/linkup/linkup_search_worker.py
+++ /dev/null
@@ -1,95 +0,0 @@
-import asyncio
-from datetime import date
-from typing import Any
-
-from linkup import LinkupClient, LinkupSourcedAnswer
-from typing_extensions import override
-
-from pipelex.cogt.search.search_setting import SearchSetting
-from pipelex.cogt.search.search_worker_abstract import SearchWorkerAbstract
-from pipelex.core.stuffs.search_result_content import SearchResultContent, SearchSourceContent
-from pipelex.hub import get_secrets_provider
-
-
-class LinkupSearchWorker(SearchWorkerAbstract):
- def __init__(self) -> None:
- api_key = get_secrets_provider().get_secret(secret_id="LINKUP_API_KEY")
- self._client = LinkupClient(api_key=api_key)
-
- def _parse_date(self, date_str: str | None) -> date | None:
- if date_str is None:
- return None
- return date.fromisoformat(date_str)
-
- @override
- async def search_sourced_answer(
- self,
- query: str,
- search_setting: SearchSetting,
- include_domains: list[str] | None = None,
- exclude_domains: list[str] | None = None,
- from_date: str | None = None,
- to_date: str | None = None,
- ) -> SearchResultContent:
- depth_value = search_setting.depth
- response: LinkupSourcedAnswer = await asyncio.to_thread(
- self._client.search,
- query=query,
- depth=depth_value, # type: ignore[arg-type]
- output_type="sourcedAnswer",
- include_images=search_setting.include_images,
- include_inline_citations=search_setting.include_inline_citations,
- max_results=search_setting.max_results,
- include_domains=include_domains,
- exclude_domains=exclude_domains,
- from_date=self._parse_date(from_date),
- to_date=self._parse_date(to_date),
- )
-
- sources: list[SearchSourceContent] = []
- for source in response.sources:
- sources.append(
- SearchSourceContent(
- name=source.name,
- url=source.url,
- snippet=source.snippet,
- )
- )
-
- return SearchResultContent(
- answer=response.answer,
- sources=sources,
- )
-
- @override
- async def search_structured(
- self,
- query: str,
- search_setting: SearchSetting,
- output_schema: type,
- include_domains: list[str] | None = None,
- exclude_domains: list[str] | None = None,
- from_date: str | None = None,
- to_date: str | None = None,
- ) -> dict[str, Any]:
- depth_value = search_setting.depth
- response = await asyncio.to_thread(
- self._client.search,
- query=query,
- depth=depth_value, # type: ignore[arg-type]
- output_type="structured",
- structured_output_schema=output_schema,
- include_images=search_setting.include_images,
- max_results=search_setting.max_results,
- include_domains=include_domains,
- exclude_domains=exclude_domains,
- from_date=self._parse_date(from_date),
- to_date=self._parse_date(to_date),
- include_sources=True,
- )
-
- # If response is a Pydantic model, convert to dict
- if hasattr(response, "model_dump"):
- result: dict[str, Any] = response.model_dump()
- return result
- return dict(response) # type: ignore[arg-type]
diff --git a/pipelex/plugins/linkup/linkup_worker.py b/pipelex/plugins/linkup/linkup_worker.py
new file mode 100644
index 000000000..8afdd80ea
--- /dev/null
+++ b/pipelex/plugins/linkup/linkup_worker.py
@@ -0,0 +1,180 @@
+from datetime import date
+from typing import Any
+
+from linkup import LinkupClient, LinkupFetchResponse, LinkupSourcedAnswer
+from typing_extensions import override
+
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.search.fetch_worker_abstract import FetchWorkerAbstract
+from pipelex.cogt.search.search_depth import SearchDepth
+from pipelex.cogt.search.search_job_factory import FetchJobFactory, SearchJobFactory
+from pipelex.cogt.search.search_setting import SearchSetting
+from pipelex.cogt.search.search_worker_abstract import SearchWorkerAbstract
+from pipelex.cogt.usage.token_category import TokenCategory
+from pipelex.core.stuffs.image_content import ImageContent
+from pipelex.core.stuffs.search_result_content import SearchResultContent, SearchSourceContent
+from pipelex.core.stuffs.text_and_images_content import TextAndImagesContent
+from pipelex.core.stuffs.text_content import TextContent
+from pipelex.hub import get_secrets_provider
+from pipelex.pipeline.job_metadata import JobMetadata, UnitJobId
+from pipelex.reporting.reporting_protocol import ReportingProtocol
+
+
+class LinkupWorker(SearchWorkerAbstract, FetchWorkerAbstract):
+ def __init__(
+ self,
+ inference_model: InferenceModelSpec,
+ reporting_delegate: ReportingProtocol | None = None,
+ ) -> None:
+ SearchWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
+ FetchWorkerAbstract.__init__(self, reporting_delegate=reporting_delegate)
+ self.inference_model = inference_model
+ api_key = get_secrets_provider().get_secret(secret_id="LINKUP_API_KEY")
+ self._linkup_client = LinkupClient(api_key=api_key)
+
+ def _parse_date(self, date_str: str | None) -> date | None:
+ if date_str is None:
+ return None
+ return date.fromisoformat(date_str)
+
+ @override
+ async def search_sourced_answer(
+ self,
+ query: str,
+ search_setting: SearchSetting,
+ job_metadata: JobMetadata,
+ include_domains: list[str] | None = None,
+ exclude_domains: list[str] | None = None,
+ from_date: str | None = None,
+ to_date: str | None = None,
+ ) -> SearchResultContent:
+ job_metadata.unit_job_id = UnitJobId.SEARCH_SOURCED_ANSWER
+ search_job = SearchJobFactory.make_search_job(job_metadata=job_metadata)
+ search_job.search_job_before_start(inference_model=self.inference_model)
+
+ depth_value = SearchDepth(self.inference_model.model_id)
+ response: LinkupSourcedAnswer = await self._linkup_client.async_search(
+ query=query,
+ depth=depth_value, # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
+ output_type="sourcedAnswer",
+ include_images=search_setting.include_images,
+ include_inline_citations=search_setting.include_inline_citations,
+ max_results=search_setting.max_results,
+ include_domains=include_domains,
+ exclude_domains=exclude_domains,
+ from_date=self._parse_date(from_date),
+ to_date=self._parse_date(to_date),
+ )
+
+ # Per-request cost model: costs are defined per million, so 1 request = 1_000_000
+ if search_tokens_usage := search_job.job_report.search_tokens_usage:
+ search_tokens_usage.nb_tokens_by_category = {TokenCategory.INPUT: 1_000_000, TokenCategory.OUTPUT: 1_000_000}
+ search_job.search_job_after_complete()
+ if self.reporting_delegate:
+ self.reporting_delegate.report_inference_job(inference_job=search_job)
+
+ sources: list[SearchSourceContent] = []
+ for source in response.sources:
+ sources.append(
+ SearchSourceContent(
+ name=source.name,
+ url=source.url,
+ snippet=source.snippet,
+ )
+ )
+
+ return SearchResultContent(
+ answer=response.answer,
+ sources=sources,
+ )
+
+ @override
+ async def search_structured(
+ self,
+ query: str,
+ search_setting: SearchSetting,
+ output_schema: type,
+ job_metadata: JobMetadata,
+ include_domains: list[str] | None = None,
+ exclude_domains: list[str] | None = None,
+ from_date: str | None = None,
+ to_date: str | None = None,
+ ) -> dict[str, Any]:
+ job_metadata.unit_job_id = UnitJobId.SEARCH_STRUCTURED
+ search_job = SearchJobFactory.make_search_job(job_metadata=job_metadata)
+ search_job.search_job_before_start(inference_model=self.inference_model)
+
+ depth_value = SearchDepth(self.inference_model.model_id)
+ response = await self._linkup_client.async_search(
+ query=query,
+ depth=depth_value, # type: ignore[arg-type] # pyright: ignore[reportArgumentType]
+ output_type="structured",
+ structured_output_schema=output_schema,
+ include_images=search_setting.include_images,
+ max_results=search_setting.max_results,
+ include_domains=include_domains,
+ exclude_domains=exclude_domains,
+ from_date=self._parse_date(from_date),
+ to_date=self._parse_date(to_date),
+ include_sources=True,
+ )
+
+ # Per-request cost model: costs are defined per million, so 1 request = 1_000_000
+ if search_tokens_usage := search_job.job_report.search_tokens_usage:
+ search_tokens_usage.nb_tokens_by_category = {TokenCategory.INPUT: 1_000_000, TokenCategory.OUTPUT: 1_000_000}
+ search_job.search_job_after_complete()
+ if self.reporting_delegate:
+ self.reporting_delegate.report_inference_job(inference_job=search_job)
+
+ # If response is a Pydantic model, convert to dict
+ if hasattr(response, "model_dump"):
+ result: dict[str, Any] = response.model_dump()
+ return result
+ return dict(response) # type: ignore[arg-type]
+
+ @override
+ async def fetch_url(
+ self,
+ url: str,
+ job_metadata: JobMetadata,
+ include_raw_html: bool | None = None,
+ render_js: bool | None = None,
+ extract_images: bool | None = None,
+ timeout: float | None = None, # noqa: ASYNC109
+ ) -> TextAndImagesContent:
+ job_metadata.unit_job_id = UnitJobId.FETCH_URL
+ fetch_job = FetchJobFactory.make_fetch_job(job_metadata=job_metadata)
+ fetch_job.fetch_job_before_start(inference_model=self.inference_model)
+
+ response: LinkupFetchResponse = await self._linkup_client.async_fetch(
+ url=url,
+ include_raw_html=include_raw_html,
+ render_js=render_js,
+ extract_images=extract_images,
+ timeout=timeout,
+ )
+
+ # Per-request cost model: costs are defined per million, so 1 request = 1_000_000
+ if fetch_tokens_usage := fetch_job.job_report.fetch_tokens_usage:
+ fetch_tokens_usage.nb_tokens_by_category = {TokenCategory.INPUT: 1_000_000, TokenCategory.OUTPUT: 1_000_000}
+ fetch_job.fetch_job_after_complete()
+ if self.reporting_delegate:
+ self.reporting_delegate.report_inference_job(inference_job=fetch_job)
+
+ text = TextContent(text=response.markdown) if response.markdown else None
+
+ images: list[ImageContent] | None = None
+ if response.images is not None:
+ images = [
+ ImageContent(
+ url=image.url,
+ caption=image.alt or None,
+ )
+ for image in response.images
+ ]
+
+ return TextAndImagesContent(
+ text=text,
+ images=images or None,
+ raw_html=response.raw_html,
+ )
diff --git a/pipelex/reporting/reporting_manager.py b/pipelex/reporting/reporting_manager.py
index 44addf246..2512dc127 100644
--- a/pipelex/reporting/reporting_manager.py
+++ b/pipelex/reporting/reporting_manager.py
@@ -12,6 +12,10 @@ from pipelex.cogt.img_gen.img_gen_report import ImgGenTokensUsage
from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_report import LLMTokensUsage
+from pipelex.cogt.search.fetch_job import FetchJob
+from pipelex.cogt.search.fetch_report import FetchTokensUsage
+from pipelex.cogt.search.search_job import SearchJob
+from pipelex.cogt.search.search_report import SearchTokensUsage
from pipelex.cogt.usage.cost_registry import CostRegistry
from pipelex.config import get_config
from pipelex.pipeline.pipeline_models import SpecialPipelineId
@@ -22,7 +26,7 @@ from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
if TYPE_CHECKING:
from pathlib import Path
-TokensUsage = LLMTokensUsage | ImgGenTokensUsage | ExtractTokensUsage
+TokensUsage = LLMTokensUsage | ImgGenTokensUsage | ExtractTokensUsage | SearchTokensUsage | FetchTokensUsage
UsageRegistryRoot = list[TokensUsage]
@@ -106,6 +110,34 @@ class ReportingManager(ReportingProtocol):
extract_token_cost_report = CostRegistry.complete_cost_report(tokens_usage=extract_tokens_usage)
log.verbose(extract_token_cost_report, title="Token Cost report")
+ def _report_search_job(self, search_job: SearchJob):
+ search_tokens_usage = search_job.job_report.search_tokens_usage
+
+ if not search_tokens_usage:
+ log.warning("Search job has no search_tokens_usage")
+ return
+
+ pipeline_run_id = search_job.job_metadata.pipeline_run_id
+ self._get_registry(pipeline_run_id).add_tokens_usage(search_tokens_usage)
+
+ if self._reporting_config.is_log_costs_to_console:
+ search_token_cost_report = CostRegistry.complete_cost_report(tokens_usage=search_tokens_usage)
+ log.verbose(search_token_cost_report, title="Token Cost report")
+
+ def _report_fetch_job(self, fetch_job: FetchJob):
+ fetch_tokens_usage = fetch_job.job_report.fetch_tokens_usage
+
+ if not fetch_tokens_usage:
+ log.warning("Fetch job has no fetch_tokens_usage")
+ return
+
+ pipeline_run_id = fetch_job.job_metadata.pipeline_run_id
+ self._get_registry(pipeline_run_id).add_tokens_usage(fetch_tokens_usage)
+
+ if self._reporting_config.is_log_costs_to_console:
+ fetch_token_cost_report = CostRegistry.complete_cost_report(tokens_usage=fetch_tokens_usage)
+ log.verbose(fetch_token_cost_report, title="Token Cost report")
+
############################################################
# ReportingProtocol
############################################################
@@ -126,6 +158,10 @@ class ReportingManager(ReportingProtocol):
self._report_img_gen_job(img_gen_job=inference_job)
elif isinstance(inference_job, ExtractJob):
self._report_extract_job(extract_job=inference_job)
+ elif isinstance(inference_job, SearchJob):
+ self._report_search_job(search_job=inference_job)
+ elif isinstance(inference_job, FetchJob):
+ self._report_fetch_job(fetch_job=inference_job)
else:
log.warning(f"ReportingManager does not support reporting for inference job type: {type(inference_job).__name__}")
diff --git a/pipelex/system/pipelex_service/pipelex_details.py b/pipelex/system/pipelex_service/pipelex_details.py
index 973c84f28..262e0920c 100644
--- a/pipelex/system/pipelex_service/pipelex_details.py
+++ b/pipelex/system/pipelex_service/pipelex_details.py
@@ -2,7 +2,7 @@ from pipelex.tools.misc.hash_utils import hash_sha256
class PipelexDetails:
- REMOTE_CONFIG_URL = "https://pipelex-config.s3.eu-west-3.amazonaws.com/pipelex_remote_config_05.json"
+ REMOTE_CONFIG_URL = "https://pipelex-config.s3.eu-west-3.amazonaws.com/pipelex_remote_config_06.json"
PIPELEX_GATEWAY_API_KEY_VAR = "PIPELEX_GATEWAY_API_KEY"
@classmethod
diff --git a/tests/e2e/pipelex/pipes/pipe_operators/pipe_search/pipe_search.mthds b/tests/e2e/pipelex/pipes/pipe_operators/pipe_search/pipe_search.mthds
index 92f5744e4..832827869 100644
--- a/tests/e2e/pipelex/pipes/pipe_operators/pipe_search/pipe_search.mthds
+++ b/tests/e2e/pipelex/pipes/pipe_operators/pipe_search/pipe_search.mthds
@@ -6,7 +6,7 @@ type = "PipeSearch"
description = "Search the web for recent news"
inputs = { topic = "Text" }
output = "SearchResult"
-model = "$linkup-standard"
+model = "$standard"
prompt = "What's the latest news about $topic in the last 24 hours?"
[pipe.search_sourced_e2e_using_preset]
@@ -14,7 +14,7 @@ type = "PipeSearch"
description = "Search the web for sourced information using the standard preset"
inputs = { topic = "Text" }
output = "SearchResult"
-model = "$linkup-standard"
+model = "$standard"
prompt = "What is $topic?"
[pipe.search_sourced_e2e_deep]
@@ -22,5 +22,5 @@ type = "PipeSearch"
description = "Search the web for sourced information using the deep preset"
inputs = { topic = "Text" }
output = "SearchResult"
-model = "$linkup-deep"
+model = "$deep"
prompt = "What are the main details about $topic?"
diff --git a/tests/integration/pipelex/cogt/test_data.py b/tests/integration/pipelex/cogt/test_data.py
index 6539f91a0..2c6d16001 100644
--- a/tests/integration/pipelex/cogt/test_data.py
+++ b/tests/integration/pipelex/cogt/test_data.py
@@ -234,18 +234,27 @@ class SerDeTestLLMCases:
]
+class FetchTestCases:
+ """Test cases for fetch integration tests."""
+
+ FETCH_URLS: ClassVar[list[tuple[str, str]]] = [ # topic, url
+ ("Wikipedia Python", "https://en.wikipedia.org/wiki/Python_(programming_language)"),
+ ]
+
+
class SearchTestCases:
"""Test cases for search integration tests."""
SOURCED_ANSWER_QUERIES: ClassVar[list[tuple[str, str]]] = [ # topic, query
- ("Middle East", "Latest events in the middle east"),
+ ("Declarative languages", "What makes declarative languages different from imperative languages?"),
+ # ("Middle East", "Latest events in the middle east"),
# ("Capital of France", "What is the capital of France?"),
# ("Python creator", "Who created the Python programming language?"),
]
STRUCTURED_QUERIES: ClassVar[list[tuple[str, str]]] = [ # topic, query
- ("Python language", "What is the Python programming language and what are its main features?"),
- ("Machine learning", "What is machine learning and its main applications?"),
+ ("Middle East", "Latest events in the middle east"),
+ # ("Python language", "What is the Python programming language and what are its main features?"),
]
diff --git a/tests/integration/pipelex/cogt/test_fetch.py b/tests/integration/pipelex/cogt/test_fetch.py
new file mode 100644
index 000000000..a470235c2
--- /dev/null
+++ b/tests/integration/pipelex/cogt/test_fetch.py
@@ -0,0 +1,64 @@
+import pytest
+
+from pipelex import pretty_print
+from pipelex.cogt.model_backends.model_type import ModelType
+from pipelex.cogt.search.search_worker_factory import SearchWorkerFactory
+from pipelex.hub import get_model_deck, get_report_delegate
+from pipelex.pipeline.job_metadata import JobMetadata
+from tests.integration.pipelex.cogt.test_data import FetchTestCases
+from tests.integration.pipelex.fixtures.model_combo import ModelCombo
+
+
[email protected](loop_scope="class")
+class TestFetch:
+ @pytest.mark.parametrize(
+ ("topic", "url"),
+ FetchTestCases.FETCH_URLS,
+ )
+ async def test_fetch_url(self, search_combo: ModelCombo, job_metadata: JobMetadata, topic: str, url: str) -> None:
+ """Verify that fetch_url returns non-empty text content."""
+ model_deck = get_model_deck()
+ inference_model = model_deck.get_required_inference_model(model_handle=search_combo.handle, model_type=ModelType.SEARCH)
+ worker = SearchWorkerFactory.make_fetch_worker(inference_model=inference_model)
+ result = await worker.fetch_url(url=url, job_metadata=job_metadata)
+ pretty_print(result, title=f"Fetch Result ({topic})")
+ assert result.text is not None, "Expected non-empty text content"
+ assert len(result.text.text) > 0, "Expected text to have content"
+ assert result.raw_html is None, "Expected raw_html to be None when not requested"
+ assert result.images is None, "Expected images to be None when not requested"
+ get_report_delegate().generate_report()
+
+ @pytest.mark.parametrize(
+ ("topic", "url"),
+ FetchTestCases.FETCH_URLS,
+ )
+ async def test_fetch_url_with_raw_html(self, search_combo: ModelCombo, job_metadata: JobMetadata, topic: str, url: str) -> None:
+ """Verify that fetch_url returns raw HTML when requested."""
+ model_deck = get_model_deck()
+ inference_model = model_deck.get_required_inference_model(model_handle=search_combo.handle, model_type=ModelType.SEARCH)
+ worker = SearchWorkerFactory.make_fetch_worker(inference_model=inference_model)
+ result = await worker.fetch_url(url=url, job_metadata=job_metadata, include_raw_html=True)
+ pretty_print(result, title=f"Fetch Result with HTML ({topic})")
+ assert result.text is not None, "Expected non-empty text content"
+ assert result.raw_html is not None, "Expected raw_html to be present when requested"
+ assert len(result.raw_html) > 0, "Expected raw_html to have content"
+ get_report_delegate().generate_report()
+
+ @pytest.mark.parametrize(
+ ("topic", "url"),
+ FetchTestCases.FETCH_URLS,
+ )
+ async def test_fetch_url_with_images(self, search_combo: ModelCombo, job_metadata: JobMetadata, topic: str, url: str) -> None:
+ """Verify that fetch_url returns images when requested."""
+ model_deck = get_model_deck()
+ inference_model = model_deck.get_required_inference_model(model_handle=search_combo.handle, model_type=ModelType.SEARCH)
+ worker = SearchWorkerFactory.make_fetch_worker(inference_model=inference_model)
+ result = await worker.fetch_url(url=url, job_metadata=job_metadata, extract_images=True)
+ pretty_print(result, title=f"Fetch Result with Images ({topic})")
+ assert result.text is not None, "Expected non-empty text content"
+ assert result.images is not None, "Expected images to be present when requested"
+ assert isinstance(result.images, list), "Expected images to be a list"
+ assert len(result.images) > 0, "Expected images list to be non-empty"
+ get_report_delegate().generate_report()
diff --git a/tests/integration/pipelex/cogt/test_search.py b/tests/integration/pipelex/cogt/test_search.py
index 82eedbbfb..4b1847e70 100644
--- a/tests/integration/pipelex/cogt/test_search.py
+++ b/tests/integration/pipelex/cogt/test_search.py
@@ -2,10 +2,13 @@ import pytest
from pydantic import BaseModel
from pipelex import pretty_print
-from pipelex.cogt.search.search_depth import SearchDepth
+from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.search.search_setting import SearchSetting
-from pipelex.cogt.search.search_worker_factory import get_search_worker
+from pipelex.cogt.search.search_worker_factory import SearchWorkerFactory
+from pipelex.hub import get_model_deck, get_report_delegate
+from pipelex.pipeline.job_metadata import JobMetadata
from tests.integration.pipelex.cogt.test_data import SearchTestCases
+from tests.integration.pipelex.fixtures.model_combo import ModelCombo
class TopicSummary(BaseModel):
@@ -22,16 +25,19 @@ class TestSearch:
("topic", "query"),
SearchTestCases.SOURCED_ANSWER_QUERIES,
)
- async def test_search_sourced_answer(self, topic: str, query: str) -> None:
+ async def test_search_sourced_answer(self, search_combo: ModelCombo, job_metadata: JobMetadata, topic: str, query: str) -> None:
"""Verify that search_sourced_answer returns a non-empty answer with sources."""
- worker = get_search_worker("linkup/standard")
+ model_deck = get_model_deck()
+ inference_model = model_deck.get_required_inference_model(model_handle=search_combo.handle, model_type=ModelType.SEARCH)
+ worker = SearchWorkerFactory.make_search_worker(inference_model=inference_model)
search_setting = SearchSetting(
- model="linkup/standard",
- depth=SearchDepth.STANDARD,
+ model=search_combo.handle,
+ max_results=2,
)
result = await worker.search_sourced_answer(
query=query,
search_setting=search_setting,
+ job_metadata=job_metadata,
)
pretty_print(result, title=f"Sourced Answer Result ({topic})")
assert result.answer, "Expected a non-empty answer"
@@ -39,25 +45,33 @@ class TestSearch:
for source in result.sources:
assert source.name, "Expected source to have a name"
assert source.url, "Expected source to have a URL"
+ get_report_delegate().generate_report()
@pytest.mark.parametrize(
("topic", "query"),
SearchTestCases.STRUCTURED_QUERIES,
)
- async def test_search_structured(self, topic: str, query: str) -> None:
+ async def test_search_structured(self, search_combo: ModelCombo, job_metadata: JobMetadata, topic: str, query: str) -> None:
"""Verify that search_structured returns a dict with the expected schema keys."""
- worker = get_search_worker("linkup/standard")
+ model_deck = get_model_deck()
+ inference_model = model_deck.get_required_inference_model(model_handle=search_combo.handle, model_type=ModelType.SEARCH)
+ worker = SearchWorkerFactory.make_search_worker(inference_model=inference_model)
search_setting = SearchSetting(
- model="linkup/standard",
- depth=SearchDepth.STANDARD,
+ model=search_combo.handle,
)
result = await worker.search_structured(
query=query,
search_setting=search_setting,
output_schema=TopicSummary,
+ job_metadata=job_metadata,
)
pretty_print(result, title=f"Structured Search Result ({topic})")
assert isinstance(result, dict), "Expected a dict result"
- assert "title" in result, "Expected 'title' key in result"
- assert "summary" in result, "Expected 'summary' key in result"
- assert "key_points" in result, "Expected 'key_points' key in result"
+ assert "data" in result, "Expected 'data' key in result"
+ assert "sources" in result, "Expected 'sources' key in result"
+ data = result["data"]
+ assert "title" in data, "Expected 'title' key in data"
+ assert "summary" in data, "Expected 'summary' key in data"
+ assert "key_points" in data, "Expected 'key_points' key in data"
+ assert len(result["sources"]) > 0, "Expected at least one source"
+ get_report_delegate().generate_report()
diff --git a/tests/integration/pipelex/conftest.py b/tests/integration/pipelex/conftest.py
index fa61fbc10..9a66f22d5 100644
--- a/tests/integration/pipelex/conftest.py
+++ b/tests/integration/pipelex/conftest.py
@@ -8,6 +8,7 @@ from .fixtures.combo_fixtures import (
extract_combo,
img_gen_combo,
llm_combo,
+ search_combo,
)
from .fixtures.extract_fixtures import (
extract_choice_for_image,
@@ -33,6 +34,7 @@ __all__ = [
"llm_combo",
"img_gen_combo",
"extract_combo",
+ "search_combo",
# Plugin fixtures
"plugin_for_openai",
"plugin_for_anthropic",
diff --git a/tests/integration/pipelex/fixtures/combo_fixtures.py b/tests/integration/pipelex/fixtures/combo_fixtures.py
index 986ff294d..1ab00b405 100644
--- a/tests/integration/pipelex/fixtures/combo_fixtures.py
+++ b/tests/integration/pipelex/fixtures/combo_fixtures.py
@@ -34,6 +34,7 @@ from tests.integration.pipelex.fixtures.model_selection import (
get_extract_combos,
get_img_gen_combos,
get_llm_combos,
+ get_search_combos,
)
@@ -150,3 +151,18 @@ def extract_combo(request: pytest.FixtureRequest):
yield combo
_cleanup_routing(monkeypatch, temp_dir)
+
+
[email protected](scope="module", params=get_search_combos())
+def search_combo(request: pytest.FixtureRequest):
+ """Provides a valid ModelCombo(handle, backend) for search with routing configured.
+
+ Yields:
+ ModelCombo(handle, backend) for search.
+ """
+ combo: ModelCombo = request.param
+ monkeypatch, temp_dir = _setup_routing_for_backend(combo.backend)
+
+ yield combo
+
+ _cleanup_routing(monkeypatch, temp_dir)
diff --git a/tests/integration/pipelex/fixtures/model_selection.py b/tests/integration/pipelex/fixtures/model_selection.py
index d5b720aba..44a96949b 100644
--- a/tests/integration/pipelex/fixtures/model_selection.py
+++ b/tests/integration/pipelex/fixtures/model_selection.py
@@ -83,3 +83,21 @@ def get_extract_combos() -> list[ModelCombo]:
)
return list(EXTRACT_COMBOS)
+
+
+@cache
+def get_search_combos() -> list[ModelCombo]:
+ """Get the list of valid (search_model, backend) combinations.
+
+ Returns:
+ List of ModelCombo(handle, backend).
+
+ Raises:
+ FileNotFoundError: If the generated fixtures file does not exist.
+ """
+ _ensure_generated_fixtures_exist()
+ from tests.integration.pipelex.fixtures._generated_model_sets import ( # noqa: PLC0415
+ SEARCH_COMBOS, # noqa: PLC2701
+ )
+
+ return list(SEARCH_COMBOS)
diff --git a/tests/unit/pipelex/cogt/models/test_model_deck.py b/tests/unit/pipelex/cogt/models/test_model_deck.py
index 0514804b2..1449b8973 100644
--- a/tests/unit/pipelex/cogt/models/test_model_deck.py
+++ b/tests/unit/pipelex/cogt/models/test_model_deck.py
@@ -8,7 +8,6 @@ from pipelex.cogt.llm.thinking_mode import ThinkingMode
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
from pipelex.cogt.model_backends.model_type import ModelType
from pipelex.cogt.models.model_deck import ModelDeck
-from pipelex.cogt.search.search_depth import SearchDepth
from pipelex.cogt.usage.cost_category import CostCategory
from pipelex.system.runtime import ProblemReaction
@@ -58,7 +57,6 @@ class TestModelDeckGetOptionalInferenceModel:
img_gen_presets={},
img_gen_choice_default="gen_image_basic",
# Search-specific
- search_default_depth=SearchDepth.STANDARD,
search_choice_default="@default-search",
model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),
)
@@ -336,7 +334,6 @@ class TestModelDeckPrefixedAliasReferences:
img_gen_presets={},
img_gen_choice_default="gen_image_basic",
# Search-specific
- search_default_depth=SearchDepth.STANDARD,
search_choice_default="@default-search",
model_deck_config=ModelDeckConfig(is_model_fallback_enabled=is_model_fallback_enabled, missing_presets_reaction=ProblemReaction.NONE),
)
@@ -490,7 +487,6 @@ class TestModelDeckGetLLMSettingWithPresets:
img_gen_presets={},
img_gen_choice_default="gen_image_basic",
# Search-specific
- search_default_depth=SearchDepth.STANDARD,
search_choice_default="@default-search",
model_deck_config=ModelDeckConfig(is_model_fallback_enabled=False, missing_presets_reaction=ProblemReaction.NONE),
)
diff --git a/tests/unit/pipelex/core/stuffs/page_content/test_data.py b/tests/unit/pipelex/core/stuffs/page_content/test_data.py
index 428d11c23..851d969b0 100644
--- a/tests/unit/pipelex/core/stuffs/page_content/test_data.py
+++ b/tests/unit/pipelex/core/stuffs/page_content/test_data.py
@@ -19,6 +19,7 @@ class TestData:
"text_and_images": {
"text": {"text": "Page content text"},
"images": None,
+ "raw_html": None,
},
"page_view": None,
}
@@ -28,6 +29,7 @@ class TestData:
"text_and_images": {
"text": {"text": "Page content text"},
"images": None,
+ "raw_html": None,
},
"page_view": {
"url": URLs.png_example_1,
@@ -42,7 +44,11 @@ class TestData:
}
# Expected outputs for render methods
- EXPECTED_RENDERED_MARKDOWN = "# text_and_images\n\n## text: ### text: Page content text\n\n## images: None\n\n# page_view: None"
- EXPECTED_RENDERED_FOR_PROMPT = "# text_and_images\n\n## text: ### text: Page content text\n\n## images: None\n\n# page_view: None"
+ EXPECTED_RENDERED_MARKDOWN = (
+ "# text_and_images\n\n## text: ### text: Page content text\n\n## images: None\n\n## raw_html: None\n\n# page_view: None"
+ )
+ EXPECTED_RENDERED_FOR_PROMPT = (
+ "# text_and_images\n\n## text: ### text: Page content text\n\n## images: None\n\n## raw_html: None\n\n# page_view: None"
+ )
# TextAndImagesContent.rendered_html returns the text content (table format, skips None values)
EXPECTED_RENDERED_HTML = "<table><tr><th>text_and_images</th><td>Page content text</td></tr></table>"
diff --git a/tests/unit/pipelex/core/stuffs/text_and_images_content/test_data.py b/tests/unit/pipelex/core/stuffs/text_and_images_content/test_data.py
index a54a44ab9..6a04c8e5f 100644
--- a/tests/unit/pipelex/core/stuffs/text_and_images_content/test_data.py
+++ b/tests/unit/pipelex/core/stuffs/text_and_images_content/test_data.py
@@ -17,6 +17,7 @@ class TestData:
EXPECTED_SMART_DUMP_TEXT_ONLY: ClassVar[dict[str, Any]] = {
"text": {"text": "Hello World"},
"images": None,
+ "raw_html": None,
}
# Expected outputs for smart_dump (text and images)
@@ -44,6 +45,7 @@ class TestData:
"filename": None,
},
],
+ "raw_html": None,
}
# Expected outputs for render methods
@@ -54,5 +56,5 @@ class TestData:
EXPECTED_RENDERED_HTML = "Hello World"
# Empty content test cases
- EXPECTED_SMART_DUMP_EMPTY: ClassVar[dict[str, Any]] = {"text": None, "images": None}
+ EXPECTED_SMART_DUMP_EMPTY: ClassVar[dict[str, Any]] = {"text": None, "images": None, "raw_html": None}
EXPECTED_RENDERED_PLAIN_EMPTY = ""