Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Add cost reporting for inference models (#735)

Louis Choquelc2986bfd89f9
diff --git a/.gitignore b/.gitignore
index 5ce54750a..b29451577 100644
--- a/.gitignore
+++ b/.gitignore
@@ -71,3 +71,6 @@ mthds-wip
# Derived artifacts (auto-generated, not tracked)
derived/
+
+# Claude plans
+.claude/plans/
\ No newline at end of file
diff --git a/.pipelex-dev/test_profiles.toml b/.pipelex-dev/test_profiles.toml
index 6d838fdb3..fc185277c 100644
--- a/.pipelex-dev/test_profiles.toml
+++ b/.pipelex-dev/test_profiles.toml
@@ -157,6 +157,7 @@ openai = [
"gpt-5.2",
"gpt-5.2-chat",
"gpt-5.2-codex",
+ "gpt-5.3-codex",
]
# --- OpenAI OSS Models ---
diff --git a/.pipelex/inference/backends/azure_openai.toml b/.pipelex/inference/backends/azure_openai.toml
index c3020daab..e4ccce3e4 100644
--- a/.pipelex/inference/backends/azure_openai.toml
+++ b/.pipelex/inference/backends/azure_openai.toml
@@ -170,6 +170,15 @@ costs = { input = 1.25, output = 10.0 }
valued_constraints = { fixed_temperature = 1 }
thinking_mode = "manual"
+# --- GPT-5.3 Series -------------------------------------------------------------
+["gpt-5.3-codex"]
+model_id = "gpt-5.3-codex-2026-02-24"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+costs = { input = 1.75, output = 14.0 }
+valued_constraints = { fixed_temperature = 1 }
+thinking_mode = "manual"
+
################################################################################
# IMAGE GENERATION MODELS
################################################################################
diff --git a/.pipelex/inference/backends/openai.toml b/.pipelex/inference/backends/openai.toml
index 3c2af8cc1..6b88e70e5 100644
--- a/.pipelex/inference/backends/openai.toml
+++ b/.pipelex/inference/backends/openai.toml
@@ -196,6 +196,14 @@ costs = { input = 1.75, output = 14.0 }
valued_constraints = { fixed_temperature = 1 }
thinking_mode = "manual"
+# --- GPT-5.3 Series -------------------------------------------------------------
+["gpt-5.3-codex"]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+costs = { input = 1.75, output = 14.0 }
+valued_constraints = { fixed_temperature = 1 }
+thinking_mode = "manual"
+
################################################################################
# IMAGE GENERATION MODELS
################################################################################
diff --git a/.pipelex/inference/backends/pipelex_gateway_models.md b/.pipelex/inference/backends/pipelex_gateway_models.md
index d519b307a..28b91b15a 100644
--- a/.pipelex/inference/backends/pipelex_gateway_models.md
+++ b/.pipelex/inference/backends/pipelex_gateway_models.md
@@ -269,6 +269,14 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
<tr>
+<td>gpt-5.3-codex</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+</tr>
+<tr>
<td>gpt-oss-120b</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
@@ -498,6 +506,12 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
<tr>
+<td>nano-banana-2</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+</tr>
+<tr>
<td>nano-banana-pro</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
@@ -508,6 +522,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T17:46:49Z
+> Last updated: 2026-03-02T23:49:25Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/.pipelex/inference/backends/pipelex_gateway_models_plain.md b/.pipelex/inference/backends/pipelex_gateway_models_plain.md
index 231b5733c..d39ef8e6b 100644
--- a/.pipelex/inference/backends/pipelex_gateway_models_plain.md
+++ b/.pipelex/inference/backends/pipelex_gateway_models_plain.md
@@ -100,6 +100,9 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **gpt-5.2-codex**
- inputs: text, images, pdf
- outputs: text, structured
+- **gpt-5.3-codex**
+ - inputs: text, images, pdf
+ - outputs: text, structured
- **gpt-oss-120b**
- inputs: text
- outputs: text, structured
@@ -184,12 +187,15 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **nano-banana**
- inputs: text, images
- outputs: image
+- **nano-banana-2**
+ - inputs: text, images
+ - outputs: image
- **nano-banana-pro**
- inputs: text, images
- outputs: image
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T17:46:49Z
+> Last updated: 2026-03-02T23:49:25Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/.pipelex/inference/deck/3_extract_deck.toml b/.pipelex/inference/deck/3_extract_deck.toml
index 2e5e5afc4..afb2e0396 100644
--- a/.pipelex/inference/deck/3_extract_deck.toml
+++ b/.pipelex/inference/deck/3_extract_deck.toml
@@ -27,8 +27,8 @@ choice_default = "@default-extract-document"
[extract.aliases]
default-premium = "azure-document-intelligence"
-default-extract-document = "mistral-document-ai-2505"
-default-extract-image = "mistral-document-ai-2505"
+default-extract-document = "azure-document-intelligence"
+default-extract-image = "azure-document-intelligence"
default-text-from-pdf = "pypdfium2-extract-pdf"
default-no-inference = "pypdfium2-extract-pdf"
diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.toml
index 12516db51..6b594bceb 100644
--- a/.pipelex/pipelex.toml
+++ b/.pipelex/pipelex.toml
@@ -65,7 +65,7 @@ pan_to_top = true # Pan to show top of graph on load
[pipelex.storage_config]
# Storage method: "local", "in_memory" (default), "s3", or "gcp"
-method = "in_memory"
+method = "local"
# Whether to fetch remote HTTP URLs and store them locally
is_fetch_remote_content_enabled = true
# Whether to upload local file paths to storage and replace with pipelex-storage:// URIs
diff --git a/pipelex/cogt/extract/extract_job.py b/pipelex/cogt/extract/extract_job.py
index 0ec341e58..b1f719bcd 100644
--- a/pipelex/cogt/extract/extract_job.py
+++ b/pipelex/cogt/extract/extract_job.py
@@ -4,7 +4,9 @@ from typing_extensions import override
from pipelex.cogt.extract.extract_input import ExtractInput
from pipelex.cogt.extract.extract_job_components import ExtractJobConfig, ExtractJobParams, ExtractJobReport
+from pipelex.cogt.extract.extract_report import ExtractTokensUsage
from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
class ExtractJob(InferenceJobAbstract):
@@ -17,12 +19,19 @@ class ExtractJob(InferenceJobAbstract):
def validate_before_execution(self):
pass
- def extract_job_before_start(self):
+ def extract_job_before_start(self, inference_model: InferenceModelSpec):
# Reset metadata
self.job_metadata.started_at = datetime.now()
# Reset outputs
self.job_report = ExtractJobReport()
+ self.job_report.extract_tokens_usage = ExtractTokensUsage(
+ job_metadata=self.job_metadata,
+ inference_model_name=inference_model.name,
+ unit_costs=inference_model.costs,
+ inference_model_id=inference_model.model_id,
+ nb_tokens_by_category={},
+ )
def extract_job_after_complete(self):
self.job_metadata.completed_at = datetime.now()
diff --git a/pipelex/cogt/extract/extract_job_components.py b/pipelex/cogt/extract/extract_job_components.py
index 8f15a4797..cf44c70ed 100644
--- a/pipelex/cogt/extract/extract_job_components.py
+++ b/pipelex/cogt/extract/extract_job_components.py
@@ -1,5 +1,6 @@
from pydantic import BaseModel
+from pipelex.cogt.extract.extract_report import ExtractTokensUsage
from pipelex.system.configuration.config_model import ConfigModel
@@ -44,4 +45,4 @@ class ExtractJobConfig(ConfigModel):
class ExtractJobReport(ConfigModel):
- pass
+ extract_tokens_usage: ExtractTokensUsage | None = None
diff --git a/pipelex/cogt/extract/extract_report.py b/pipelex/cogt/extract/extract_report.py
new file mode 100644
index 000000000..55e803ab3
--- /dev/null
+++ b/pipelex/cogt/extract/extract_report.py
@@ -0,0 +1,72 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import StrEnum
+
+
+class ExtractTokenCostReportField(StrEnum):
+ MODEL_TYPE = "model_type"
+ EXTRACT_NAME = "extract_name"
+ PLATFORM_EXTRACT_ID = "platform_extract_id"
+ NB_TOKENS_INPUT = "nb_tokens_input"
+ NB_TOKENS_INPUT_CACHED = "nb_tokens_input_cached"
+ NB_TOKENS_INPUT_NON_CACHED = "nb_tokens_input_non_cached"
+ NB_TOKENS_INPUT_JOINED = "nb_tokens_input_joined"
+ NB_TOKENS_OUTPUT = "nb_tokens_output"
+ COST_INPUT_CACHED = "cost_input_cached"
+ COST_INPUT_NON_CACHED = "cost_input_non_cached"
+ COST_INPUT_JOINED = "cost_input_joined"
+ COST_OUTPUT = "cost_output"
+
+ @staticmethod
+ def report_field_for_nb_tokens_by_category(token_category: TokenCategory) -> str:
+ return f"nb_tokens_{token_category}"
+
+ @staticmethod
+ def report_field_for_cost_by_category(token_category: CostCategory) -> str:
+ return f"cost_{token_category}"
+
+
+class ExtractTokenCostReport(BaseModel):
+ model_type: str = "extract"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ platform_model_id: str
+
+ nb_tokens_by_category: NbTokensByCategoryDict
+ costs_by_token_category: CostsByCategoryDict
+
+ def as_flat_dictionary(self) -> dict[str, Any]:
+ the_dict: dict[str, Any] = {}
+ dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)
+ the_dict.update(dict_for_job_metadata)
+ dict_for_model: dict[str, Any] = {
+ ExtractTokenCostReportField.MODEL_TYPE: self.model_type,
+ ExtractTokenCostReportField.EXTRACT_NAME: self.inference_model_name,
+ ExtractTokenCostReportField.PLATFORM_EXTRACT_ID: self.platform_model_id,
+ }
+ the_dict.update(dict_for_model)
+ dict_for_nb_tokens = {
+ ExtractTokenCostReportField.report_field_for_nb_tokens_by_category(token_category): nb_tokens
+ for token_category, nb_tokens in self.nb_tokens_by_category.items()
+ }
+ the_dict.update(dict_for_nb_tokens)
+ dict_for_costs = {
+ ExtractTokenCostReportField.report_field_for_cost_by_category(token_category): cost
+ for token_category, cost in self.costs_by_token_category.items()
+ }
+ the_dict.update(dict_for_costs)
+ return the_dict
+
+
+class ExtractTokensUsage(BaseModel):
+ model_type: str = "extract"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ unit_costs: CostsByCategoryDict
+ inference_model_id: str
+ nb_tokens_by_category: NbTokensByCategoryDict
diff --git a/pipelex/cogt/extract/extract_worker_abstract.py b/pipelex/cogt/extract/extract_worker_abstract.py
index 6f1354bd6..cacfe6450 100644
--- a/pipelex/cogt/extract/extract_worker_abstract.py
+++ b/pipelex/cogt/extract/extract_worker_abstract.py
@@ -9,6 +9,7 @@ from pipelex.cogt.extract.extract_job import ExtractJob
from pipelex.cogt.extract.extract_output import ExtractOutput
from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
from pipelex.pipeline.job_metadata import UnitJobId
from pipelex.reporting.reporting_protocol import ReportingProtocol
@@ -78,11 +79,19 @@ class ExtractWorkerAbstract(InferenceWorkerAbstract):
extract_job.job_metadata.unit_job_id = UnitJobId.EXTRACT_PAGES
# Prepare job
- extract_job.extract_job_before_start()
+ extract_job.extract_job_before_start(inference_model=self.inference_model)
# Execute job
result = await self._extract_pages(extract_job=extract_job)
+ # Populate page count as fallback usage (only if no real usage was reported)
+ if (extract_tokens_usage := extract_job.job_report.extract_tokens_usage) and not extract_tokens_usage.nb_tokens_by_category:
+ nb_tokens: NbTokensByCategoryDict = {
+ TokenCategory.INPUT: len(result.pages) * 1_000_000,
+ TokenCategory.OUTPUT: len(result.pages) * 1_000_000,
+ }
+ extract_tokens_usage.nb_tokens_by_category = nb_tokens
+
# Report job
extract_job.extract_job_after_complete()
if self.reporting_delegate:
diff --git a/pipelex/cogt/img_gen/img_gen_job.py b/pipelex/cogt/img_gen/img_gen_job.py
index a8a8cf81c..bc01a867d 100644
--- a/pipelex/cogt/img_gen/img_gen_job.py
+++ b/pipelex/cogt/img_gen/img_gen_job.py
@@ -4,7 +4,9 @@ from typing_extensions import override
from pipelex.cogt.img_gen.img_gen_job_components import ImgGenJobConfig, ImgGenJobParams, ImgGenJobReport
from pipelex.cogt.img_gen.img_gen_prompt import ImgGenPrompt
+from pipelex.cogt.img_gen.img_gen_report import ImgGenTokensUsage
from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
+from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
class ImgGenJob(InferenceJobAbstract):
@@ -17,12 +19,19 @@ class ImgGenJob(InferenceJobAbstract):
def validate_before_execution(self):
self.img_gen_prompt.validate_before_execution()
- def img_gen_job_before_start(self):
+ def img_gen_job_before_start(self, inference_model: InferenceModelSpec):
# Reset metadata
self.job_metadata.started_at = datetime.now()
# Reset outputs
self.job_report = ImgGenJobReport()
+ self.job_report.img_gen_tokens_usage = ImgGenTokensUsage(
+ job_metadata=self.job_metadata,
+ inference_model_name=inference_model.name,
+ unit_costs=inference_model.costs,
+ inference_model_id=inference_model.model_id,
+ nb_tokens_by_category={},
+ )
def img_gen_job_after_complete(self):
self.job_metadata.completed_at = datetime.now()
diff --git a/pipelex/cogt/img_gen/img_gen_job_components.py b/pipelex/cogt/img_gen/img_gen_job_components.py
index bd3c201d8..255834a24 100644
--- a/pipelex/cogt/img_gen/img_gen_job_components.py
+++ b/pipelex/cogt/img_gen/img_gen_job_components.py
@@ -2,6 +2,7 @@ from typing import Literal
from pydantic import BaseModel, Field, model_validator
+from pipelex.cogt.img_gen.img_gen_report import ImgGenTokensUsage
from pipelex.system.configuration.config_model import ConfigModel
from pipelex.tools.misc.image_utils import ImageFormat
from pipelex.types import Self, StrEnum
@@ -111,4 +112,4 @@ class ImgGenJobConfig(ConfigModel):
class ImgGenJobReport(ConfigModel):
- pass
+ img_gen_tokens_usage: ImgGenTokensUsage | None = None
diff --git a/pipelex/cogt/img_gen/img_gen_report.py b/pipelex/cogt/img_gen/img_gen_report.py
new file mode 100644
index 000000000..7390df79a
--- /dev/null
+++ b/pipelex/cogt/img_gen/img_gen_report.py
@@ -0,0 +1,72 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
+from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.types import StrEnum
+
+
+class ImgGenTokenCostReportField(StrEnum):
+ MODEL_TYPE = "model_type"
+ IMG_GEN_NAME = "img_gen_name"
+ PLATFORM_IMG_GEN_ID = "platform_img_gen_id"
+ NB_TOKENS_INPUT = "nb_tokens_input"
+ NB_TOKENS_INPUT_CACHED = "nb_tokens_input_cached"
+ NB_TOKENS_INPUT_NON_CACHED = "nb_tokens_input_non_cached"
+ NB_TOKENS_INPUT_JOINED = "nb_tokens_input_joined"
+ NB_TOKENS_OUTPUT = "nb_tokens_output"
+ COST_INPUT_CACHED = "cost_input_cached"
+ COST_INPUT_NON_CACHED = "cost_input_non_cached"
+ COST_INPUT_JOINED = "cost_input_joined"
+ COST_OUTPUT = "cost_output"
+
+ @staticmethod
+ def report_field_for_nb_tokens_by_category(token_category: TokenCategory) -> str:
+ return f"nb_tokens_{token_category}"
+
+ @staticmethod
+ def report_field_for_cost_by_category(token_category: CostCategory) -> str:
+ return f"cost_{token_category}"
+
+
+class ImgGenTokenCostReport(BaseModel):
+ model_type: str = "img_gen"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ platform_model_id: str
+
+ nb_tokens_by_category: NbTokensByCategoryDict
+ costs_by_token_category: CostsByCategoryDict
+
+ def as_flat_dictionary(self) -> dict[str, Any]:
+ the_dict: dict[str, Any] = {}
+ dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)
+ the_dict.update(dict_for_job_metadata)
+ dict_for_model: dict[str, Any] = {
+ ImgGenTokenCostReportField.MODEL_TYPE: self.model_type,
+ ImgGenTokenCostReportField.IMG_GEN_NAME: self.inference_model_name,
+ ImgGenTokenCostReportField.PLATFORM_IMG_GEN_ID: self.platform_model_id,
+ }
+ the_dict.update(dict_for_model)
+ dict_for_nb_tokens = {
+ ImgGenTokenCostReportField.report_field_for_nb_tokens_by_category(token_category): nb_tokens
+ for token_category, nb_tokens in self.nb_tokens_by_category.items()
+ }
+ the_dict.update(dict_for_nb_tokens)
+ dict_for_costs = {
+ ImgGenTokenCostReportField.report_field_for_cost_by_category(token_category): cost
+ for token_category, cost in self.costs_by_token_category.items()
+ }
+ the_dict.update(dict_for_costs)
+ return the_dict
+
+
+class ImgGenTokensUsage(BaseModel):
+ model_type: str = "img_gen"
+ job_metadata: JobMetadata
+ inference_model_name: str
+ unit_costs: CostsByCategoryDict
+ inference_model_id: str
+ nb_tokens_by_category: NbTokensByCategoryDict
diff --git a/pipelex/cogt/img_gen/img_gen_worker_abstract.py b/pipelex/cogt/img_gen/img_gen_worker_abstract.py
index d05408fcc..86cd0cbb8 100644
--- a/pipelex/cogt/img_gen/img_gen_worker_abstract.py
+++ b/pipelex/cogt/img_gen/img_gen_worker_abstract.py
@@ -54,7 +54,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):
img_gen_job.job_metadata.unit_job_id = UnitJobId.IMG_GEN_TEXT_TO_IMAGE
# Prepare job
- img_gen_job.img_gen_job_before_start()
+ img_gen_job.img_gen_job_before_start(inference_model=self.inference_model)
# Execute job
result = await self._gen_image(img_gen_job=img_gen_job)
@@ -90,7 +90,7 @@ class ImgGenWorkerAbstract(InferenceWorkerAbstract):
img_gen_job.job_metadata.unit_job_id = UnitJobId.IMG_GEN_TEXT_TO_IMAGE
# Prepare job
- img_gen_job.img_gen_job_before_start()
+ img_gen_job.img_gen_job_before_start(inference_model=self.inference_model)
# Execute job
result = await self._gen_image_list(img_gen_job=img_gen_job, nb_images=nb_images)
diff --git a/pipelex/cogt/llm/llm_report.py b/pipelex/cogt/llm/llm_report.py
index 24c29ec77..eb27f1dec 100644
--- a/pipelex/cogt/llm/llm_report.py
+++ b/pipelex/cogt/llm/llm_report.py
@@ -9,6 +9,7 @@ from pipelex.types import StrEnum
class LLMTokenCostReportField(StrEnum):
+ MODEL_TYPE = "model_type"
LLM_NAME = "llm_name"
PLATFORM_LLM_ID = "platform_llm_id"
NB_TOKENS_INPUT = "nb_tokens_input"
@@ -31,6 +32,7 @@ class LLMTokenCostReportField(StrEnum):
class LLMTokenCostReport(BaseModel):
+ model_type: str = "llm"
job_metadata: JobMetadata
inference_model_name: str
platform_llm_id: str
@@ -43,6 +45,7 @@ class LLMTokenCostReport(BaseModel):
dict_for_job_metadata = self.job_metadata.model_dump(serialize_as_any=True)
the_dict.update(dict_for_job_metadata)
dict_for_llm: dict[str, Any] = {
+ LLMTokenCostReportField.MODEL_TYPE: self.model_type,
LLMTokenCostReportField.LLM_NAME: self.inference_model_name,
LLMTokenCostReportField.PLATFORM_LLM_ID: self.platform_llm_id,
}
@@ -61,6 +64,7 @@ class LLMTokenCostReport(BaseModel):
class LLMTokensUsage(BaseModel):
+ model_type: str = "llm"
job_metadata: JobMetadata
inference_model_name: str
unit_costs: CostsByCategoryDict
diff --git a/pipelex/cogt/usage/cost_registry.py b/pipelex/cogt/usage/cost_registry.py
index 6b17b046f..6d523ca9c 100644
--- a/pipelex/cogt/usage/cost_registry.py
+++ b/pipelex/cogt/usage/cost_registry.py
@@ -1,4 +1,5 @@
import csv
+from collections.abc import Sequence
from pathlib import Path
from typing import Any
@@ -8,6 +9,8 @@ from rich.table import Table
from pipelex import log
from pipelex.cogt.exceptions import CostRegistryError
+from pipelex.cogt.extract.extract_report import ExtractTokenCostReport, ExtractTokenCostReportField, ExtractTokensUsage
+from pipelex.cogt.img_gen.img_gen_report import ImgGenTokenCostReport, ImgGenTokenCostReportField, ImgGenTokensUsage
from pipelex.cogt.llm.llm_report import LLMTokenCostReport, LLMTokenCostReportField, LLMTokensUsage
from pipelex.cogt.usage.cost_category import CostCategory, CostsByCategoryDict
from pipelex.cogt.usage.costs_per_token import model_cost_per_token
@@ -15,7 +18,9 @@ from pipelex.cogt.usage.token_category import TokenCategory
from pipelex.hub import get_console
from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
-CostRegistryRoot = list[LLMTokenCostReport]
+TokensUsage = LLMTokensUsage | ImgGenTokensUsage | ExtractTokensUsage
+TokenCostReport = LLMTokenCostReport | ImgGenTokenCostReport | ExtractTokenCostReport
+CostRegistryRoot = list[TokenCostReport]
class CostRegistry(RootModel[CostRegistryRoot]):
@@ -33,73 +38,83 @@ class CostRegistry(RootModel[CostRegistryRoot]):
def generate_report(
cls,
pipeline_run_id: str,
- llm_tokens_usages: list[LLMTokensUsage],
+ tokens_usages: Sequence[TokensUsage],
unit_scale: float,
cost_report_file_path: Path | None = None,
):
- if not llm_tokens_usages:
+ if not tokens_usages:
if pipeline_run_id != "untitled":
log.warning(f"No report to generate for pipeline '{pipeline_run_id}'")
else:
log.verbose(f"No report to generate for pipeline '{pipeline_run_id}'")
return
cost_registry = CostRegistry()
- for llm_tokens_usage in llm_tokens_usages:
- cost_report = cls.complete_cost_report(llm_tokens_usage=llm_tokens_usage)
+ for tokens_usage in tokens_usages:
+ cost_report = cls.complete_cost_report(tokens_usage=tokens_usage)
cost_registry.root.append(cost_report)
records = cost_registry.to_records()
+ # Use LLMTokenCostReportField for field names (same string values as ImgGenTokenCostReportField)
+ report_field = LLMTokenCostReportField
+
# Calculate total costs overall
- total_nb_tokens_input_cached = sum(record[LLMTokenCostReportField.NB_TOKENS_INPUT_CACHED] for record in records)
- total_nb_tokens_input_non_cached = sum(record[LLMTokenCostReportField.NB_TOKENS_INPUT_NON_CACHED] for record in records)
- total_nb_tokens_input_joined = sum(record[LLMTokenCostReportField.NB_TOKENS_INPUT_JOINED] for record in records)
- total_nb_tokens_output = sum(record[LLMTokenCostReportField.NB_TOKENS_OUTPUT] for record in records)
- total_cost_input_cached = sum(record[LLMTokenCostReportField.COST_INPUT_CACHED] for record in records)
- total_cost_input_non_cached = sum(record[LLMTokenCostReportField.COST_INPUT_NON_CACHED] for record in records)
- total_cost_input_joined = sum(record[LLMTokenCostReportField.COST_INPUT_JOINED] for record in records)
- total_cost_output = sum(record[LLMTokenCostReportField.COST_OUTPUT] for record in records)
+ total_nb_tokens_input_cached = sum(record.get(report_field.NB_TOKENS_INPUT_CACHED, 0) for record in records)
+ total_nb_tokens_input_non_cached = sum(record.get(report_field.NB_TOKENS_INPUT_NON_CACHED, 0) for record in records)
+ total_nb_tokens_input_joined = sum(record.get(report_field.NB_TOKENS_INPUT_JOINED, 0) for record in records)
+ total_nb_tokens_output = sum(record.get(report_field.NB_TOKENS_OUTPUT, 0) for record in records)
+ total_cost_input_cached = sum(record.get(report_field.COST_INPUT_CACHED, 0) for record in records)
+ total_cost_input_non_cached = sum(record.get(report_field.COST_INPUT_NON_CACHED, 0) for record in records)
+ total_cost_input_joined = sum(record.get(report_field.COST_INPUT_JOINED, 0) for record in records)
+ total_cost_output = sum(record.get(report_field.COST_OUTPUT, 0) for record in records)
total_cost = cls.compute_total_cost(
input_non_cached_cost=total_cost_input_non_cached,
input_cached_cost=total_cost_input_cached,
output_cost=total_cost_output,
)
- # Calculate costs per LLM model - group by LLM name
- grouped_by_llm: dict[str, dict[str, float]] = {}
+ # Calculate costs per model - group by model name
+ grouped_by_model: dict[str, dict[str, float]] = {}
+ model_types: dict[str, str] = {}
for record in records:
- llm_name = record[LLMTokenCostReportField.LLM_NAME]
- if llm_name not in grouped_by_llm:
- grouped_by_llm[llm_name] = {
- LLMTokenCostReportField.NB_TOKENS_INPUT_CACHED: 0,
- LLMTokenCostReportField.NB_TOKENS_INPUT_NON_CACHED: 0,
- LLMTokenCostReportField.NB_TOKENS_INPUT_JOINED: 0,
- LLMTokenCostReportField.NB_TOKENS_OUTPUT: 0,
- LLMTokenCostReportField.COST_INPUT_CACHED: 0.0,
- LLMTokenCostReportField.COST_INPUT_NON_CACHED: 0.0,
- LLMTokenCostReportField.COST_INPUT_JOINED: 0.0,
- LLMTokenCostReportField.COST_OUTPUT: 0.0,
+ model_name = (
+ record.get(report_field.LLM_NAME)
+ or record.get(ImgGenTokenCostReportField.IMG_GEN_NAME)
+ or record.get(ExtractTokenCostReportField.EXTRACT_NAME, "unknown")
+ )
+ model_type = record.get(report_field.MODEL_TYPE, "llm")
+ model_types[model_name] = model_type
+ if model_name not in grouped_by_model:
+ grouped_by_model[model_name] = {
+ report_field.NB_TOKENS_INPUT_CACHED: 0,
+ report_field.NB_TOKENS_INPUT_NON_CACHED: 0,
+ report_field.NB_TOKENS_INPUT_JOINED: 0,
+ report_field.NB_TOKENS_OUTPUT: 0,
+ report_field.COST_INPUT_CACHED: 0.0,
+ report_field.COST_INPUT_NON_CACHED: 0.0,
+ report_field.COST_INPUT_JOINED: 0.0,
+ report_field.COST_OUTPUT: 0.0,
}
# Aggregate values
for field in [
- LLMTokenCostReportField.NB_TOKENS_INPUT_CACHED,
- LLMTokenCostReportField.NB_TOKENS_INPUT_NON_CACHED,
- LLMTokenCostReportField.NB_TOKENS_INPUT_JOINED,
- LLMTokenCostReportField.NB_TOKENS_OUTPUT,
- LLMTokenCostReportField.COST_INPUT_CACHED,
- LLMTokenCostReportField.COST_INPUT_NON_CACHED,
- LLMTokenCostReportField.COST_INPUT_JOINED,
- LLMTokenCostReportField.COST_OUTPUT,
+ report_field.NB_TOKENS_INPUT_CACHED,
+ report_field.NB_TOKENS_INPUT_NON_CACHED,
+ report_field.NB_TOKENS_INPUT_JOINED,
+ report_field.NB_TOKENS_OUTPUT,
+ report_field.COST_INPUT_CACHED,
+ report_field.COST_INPUT_NON_CACHED,
+ report_field.COST_INPUT_JOINED,
+ report_field.COST_OUTPUT,
]:
- grouped_by_llm[llm_name][field] += record[field]
+ grouped_by_model[model_name][field] += record.get(field, 0)
- if not grouped_by_llm:
- msg = "Empty report aggregation by LLM name"
+ if not grouped_by_model:
+ msg = "Empty report aggregation by model name"
raise CostRegistryError(msg)
console = get_console()
- title = f"Costs by LLM model for pipeline '{pipeline_run_id}'"
+ title = f"Costs by model for pipeline '{pipeline_run_id}'"
table = Table(title=title, box=box.ROUNDED)
scale_str: str
@@ -109,6 +124,7 @@ class CostRegistry(RootModel[CostRegistryRoot]):
scale_str = str(unit_scale)
# Add columns
table.add_column("Model", style="cyan", overflow="fold", width=30)
+ table.add_column("Type", style="dim cyan", width=8)
table.add_column("Input Cached", justify="right", style="green")
table.add_column("Input Non Cached", justify="right", style="green")
table.add_column("Input Joined", justify="right", style="green")
@@ -119,23 +135,24 @@ class CostRegistry(RootModel[CostRegistryRoot]):
table.add_column(f"Output Cost ({scale_str}$)", justify="right", style="yellow")
table.add_column(f"Total Cost ({scale_str}$)", justify="right", style="bold yellow")
- # Add rows for each LLM model
- for llm_name, aggregated_data in grouped_by_llm.items():
+ # Add rows for each model
+ for model_name, aggregated_data in grouped_by_model.items():
row_total_cost = cls.compute_total_cost(
- input_non_cached_cost=aggregated_data[LLMTokenCostReportField.COST_INPUT_NON_CACHED],
- input_cached_cost=aggregated_data[LLMTokenCostReportField.COST_INPUT_CACHED],
- output_cost=aggregated_data[LLMTokenCostReportField.COST_OUTPUT],
+ input_non_cached_cost=aggregated_data[report_field.COST_INPUT_NON_CACHED],
+ input_cached_cost=aggregated_data[report_field.COST_INPUT_CACHED],
+ output_cost=aggregated_data[report_field.COST_OUTPUT],
)
table.add_row(
- llm_name,
- f"{int(aggregated_data[LLMTokenCostReportField.NB_TOKENS_INPUT_CACHED]):,}",
- f"{int(aggregated_data[LLMTokenCostReportField.NB_TOKENS_INPUT_NON_CACHED]):,}",
- f"{int(aggregated_data[LLMTokenCostReportField.NB_TOKENS_INPUT_JOINED]):,}",
- f"{int(aggregated_data[LLMTokenCostReportField.NB_TOKENS_OUTPUT]):,}",
- f"{aggregated_data[LLMTokenCostReportField.COST_INPUT_CACHED] / unit_scale:.4f}",
- f"{aggregated_data[LLMTokenCostReportField.COST_INPUT_NON_CACHED] / unit_scale:.4f}",
- f"{aggregated_data[LLMTokenCostReportField.COST_INPUT_JOINED] / unit_scale:.4f}",
- f"{aggregated_data[LLMTokenCostReportField.COST_OUTPUT] / unit_scale:.4f}",
+ model_name,
+ model_types.get(model_name, "llm"),
+ f"{int(aggregated_data[report_field.NB_TOKENS_INPUT_CACHED]):,}",
+ f"{int(aggregated_data[report_field.NB_TOKENS_INPUT_NON_CACHED]):,}",
+ f"{int(aggregated_data[report_field.NB_TOKENS_INPUT_JOINED]):,}",
+ f"{int(aggregated_data[report_field.NB_TOKENS_OUTPUT]):,}",
+ f"{aggregated_data[report_field.COST_INPUT_CACHED] / unit_scale:.4f}",
+ f"{aggregated_data[report_field.COST_INPUT_NON_CACHED] / unit_scale:.4f}",
+ f"{aggregated_data[report_field.COST_INPUT_JOINED] / unit_scale:.4f}",
+ f"{aggregated_data[report_field.COST_OUTPUT] / unit_scale:.4f}",
f"{row_total_cost / unit_scale:.4f}",
)
@@ -143,6 +160,7 @@ class CostRegistry(RootModel[CostRegistryRoot]):
footer_style = "bold"
table.add_row(
"Total",
+ "",
f"{total_nb_tokens_input_cached:,}",
f"{total_nb_tokens_input_non_cached:,}",
f"{total_nb_tokens_input_joined:,}",
@@ -157,7 +175,7 @@ class CostRegistry(RootModel[CostRegistryRoot]):
)
console.print(table)
- console.print(" [dim]Note: the report only includes LLMs and some costs might be missing or not up-to-date.[/dim]")
+ console.print(" [dim]Note: some costs might be missing or not up-to-date.[/dim]")
if cost_report_file_path:
cls.save_to_csv(records, cost_report_file_path)
@@ -186,25 +204,45 @@ class CostRegistry(RootModel[CostRegistryRoot]):
return input_non_cached_cost + input_cached_cost + output_cost
@classmethod
- def compute_cost_report(cls, llm_tokens_usage: LLMTokensUsage) -> LLMTokenCostReport:
+ def compute_cost_report(cls, tokens_usage: TokensUsage) -> TokenCostReport:
costs_by_token_category: CostsByCategoryDict = {}
- for token_type, nb_tokens in llm_tokens_usage.nb_tokens_by_category.items():
+ for token_type, nb_tokens in tokens_usage.nb_tokens_by_category.items():
cost_per_token = model_cost_per_token(
- costs=llm_tokens_usage.unit_costs,
+ costs=tokens_usage.unit_costs,
cost_category=token_type.to_cost_category,
)
costs_by_token_category[token_type.to_cost_category] = cost_per_token * nb_tokens
- return LLMTokenCostReport(
- job_metadata=llm_tokens_usage.job_metadata,
- inference_model_name=llm_tokens_usage.inference_model_name,
- platform_llm_id=llm_tokens_usage.inference_model_id,
- nb_tokens_by_category=llm_tokens_usage.nb_tokens_by_category,
+
+ if isinstance(tokens_usage, LLMTokensUsage):
+ return LLMTokenCostReport(
+ model_type=tokens_usage.model_type,
+ job_metadata=tokens_usage.job_metadata,
+ inference_model_name=tokens_usage.inference_model_name,
+ platform_llm_id=tokens_usage.inference_model_id,
+ nb_tokens_by_category=tokens_usage.nb_tokens_by_category,
+ costs_by_token_category=costs_by_token_category,
+ )
+ if isinstance(tokens_usage, ImgGenTokensUsage):
+ return ImgGenTokenCostReport(
+ model_type=tokens_usage.model_type,
+ job_metadata=tokens_usage.job_metadata,
+ inference_model_name=tokens_usage.inference_model_name,
+ platform_model_id=tokens_usage.inference_model_id,
+ nb_tokens_by_category=tokens_usage.nb_tokens_by_category,
+ costs_by_token_category=costs_by_token_category,
+ )
+ return ExtractTokenCostReport(
+ model_type=tokens_usage.model_type,
+ job_metadata=tokens_usage.job_metadata,
+ inference_model_name=tokens_usage.inference_model_name,
+ platform_model_id=tokens_usage.inference_model_id,
+ nb_tokens_by_category=tokens_usage.nb_tokens_by_category,
costs_by_token_category=costs_by_token_category,
)
@classmethod
- def complete_cost_report(cls, llm_tokens_usage: LLMTokensUsage) -> LLMTokenCostReport:
- cost_report = cls.compute_cost_report(llm_tokens_usage=llm_tokens_usage)
+ def complete_cost_report(cls, tokens_usage: TokensUsage) -> TokenCostReport:
+ cost_report = cls.compute_cost_report(tokens_usage=tokens_usage)
# compute the input_non_cached tokens
if cost_report.nb_tokens_by_category.get(TokenCategory.INPUT_NON_CACHED) is not None:
msg = "CostCategory.INPUT_NON_CACHED already exists in the cost report"
@@ -220,7 +258,7 @@ class CostRegistry(RootModel[CostRegistryRoot]):
cost_report.nb_tokens_by_category[TokenCategory.INPUT_CACHED] = nb_tokens_input_cached
cost_report.costs_by_token_category[CostCategory.INPUT_NON_CACHED] = nb_tokens_input_non_cached * model_cost_per_token(
- costs=llm_tokens_usage.unit_costs,
+ costs=tokens_usage.unit_costs,
cost_category=CostCategory.INPUT_NON_CACHED,
)
costs_input_cached = cost_report.costs_by_token_category.get(CostCategory.INPUT_CACHED, 0)
diff --git a/pipelex/kit/configs/inference/backends/azure_openai.toml b/pipelex/kit/configs/inference/backends/azure_openai.toml
index c3020daab..e4ccce3e4 100644
--- a/pipelex/kit/configs/inference/backends/azure_openai.toml
+++ b/pipelex/kit/configs/inference/backends/azure_openai.toml
@@ -170,6 +170,15 @@ costs = { input = 1.25, output = 10.0 }
valued_constraints = { fixed_temperature = 1 }
thinking_mode = "manual"
+# --- GPT-5.3 Series -------------------------------------------------------------
+["gpt-5.3-codex"]
+model_id = "gpt-5.3-codex-2026-02-24"
+inputs = ["text", "images"]
+outputs = ["text", "structured"]
+costs = { input = 1.75, output = 14.0 }
+valued_constraints = { fixed_temperature = 1 }
+thinking_mode = "manual"
+
################################################################################
# IMAGE GENERATION MODELS
################################################################################
diff --git a/pipelex/kit/configs/inference/backends/openai.toml b/pipelex/kit/configs/inference/backends/openai.toml
index 3c2af8cc1..6b88e70e5 100644
--- a/pipelex/kit/configs/inference/backends/openai.toml
+++ b/pipelex/kit/configs/inference/backends/openai.toml
@@ -196,6 +196,14 @@ costs = { input = 1.75, output = 14.0 }
valued_constraints = { fixed_temperature = 1 }
thinking_mode = "manual"
+# --- GPT-5.3 Series -------------------------------------------------------------
+["gpt-5.3-codex"]
+inputs = ["text", "images", "pdf"]
+outputs = ["text", "structured"]
+costs = { input = 1.75, output = 14.0 }
+valued_constraints = { fixed_temperature = 1 }
+thinking_mode = "manual"
+
################################################################################
# IMAGE GENERATION MODELS
################################################################################
diff --git a/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md b/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
index d519b307a..28b91b15a 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
+++ b/pipelex/kit/configs/inference/backends/pipelex_gateway_models.md
@@ -269,6 +269,14 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
<tr>
+<td>gpt-5.3-codex</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+</tr>
+<tr>
<td>gpt-oss-120b</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">❌</td>
@@ -498,6 +506,12 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
</tr>
<tr>
+<td>nano-banana-2</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
+<td style="text-align:center;background-color:rgba(76,175,80,0.15)">✅</td>
+</tr>
+<tr>
<td>nano-banana-pro</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
<td style="text-align:center;background-color:rgba(33,150,243,0.15)">✅</td>
@@ -508,6 +522,6 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T17:46:49Z
+> Last updated: 2026-03-02T23:49:25Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md b/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
index 231b5733c..d39ef8e6b 100644
--- a/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
+++ b/pipelex/kit/configs/inference/backends/pipelex_gateway_models_plain.md
@@ -100,6 +100,9 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **gpt-5.2-codex**
- inputs: text, images, pdf
- outputs: text, structured
+- **gpt-5.3-codex**
+ - inputs: text, images, pdf
+ - outputs: text, structured
- **gpt-oss-120b**
- inputs: text
- outputs: text, structured
@@ -184,12 +187,15 @@ For configuration details, see the [documentation](https://docs.pipelex.com/late
- **nano-banana**
- inputs: text, images
- outputs: image
+- **nano-banana-2**
+ - inputs: text, images
+ - outputs: image
- **nano-banana-pro**
- inputs: text, images
- outputs: image
> **AUTO-GENERATED FILE** - Do not edit manually.
-> Last updated: 2026-03-02T17:46:49Z
+> Last updated: 2026-03-02T23:49:25Z
>
> Run `pipelex-dev update-gateway-models` or `make ugm` to regenerate.
\ No newline at end of file
diff --git a/pipelex/kit/configs/inference/deck/3_extract_deck.toml b/pipelex/kit/configs/inference/deck/3_extract_deck.toml
index 2e5e5afc4..afb2e0396 100644
--- a/pipelex/kit/configs/inference/deck/3_extract_deck.toml
+++ b/pipelex/kit/configs/inference/deck/3_extract_deck.toml
@@ -27,8 +27,8 @@ choice_default = "@default-extract-document"
[extract.aliases]
default-premium = "azure-document-intelligence"
-default-extract-document = "mistral-document-ai-2505"
-default-extract-image = "mistral-document-ai-2505"
+default-extract-document = "azure-document-intelligence"
+default-extract-image = "azure-document-intelligence"
default-text-from-pdf = "pypdfium2-extract-pdf"
default-no-inference = "pypdfium2-extract-pdf"
diff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.toml
index 12516db51..6b594bceb 100644
--- a/pipelex/kit/configs/pipelex.toml
+++ b/pipelex/kit/configs/pipelex.toml
@@ -65,7 +65,7 @@ pan_to_top = true # Pan to show top of graph on load
[pipelex.storage_config]
# Storage method: "local", "in_memory" (default), "s3", or "gcp"
-method = "in_memory"
+method = "local"
# Whether to fetch remote HTTP URLs and store them locally
is_fetch_remote_content_enabled = true
# Whether to upload local file paths to storage and replace with pipelex-storage:// URIs
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 00d6ccbfb..f15d4c47f 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -5,7 +5,7 @@
[pipelex.storage_config]
is_fetch_remote_content_enabled = true
is_upload_local_content_enabled = true
-method = "in_memory" # local, in_memory, s3, gcp
+method = "local" # local, in_memory, s3, gcp
[pipelex.storage_config.local]
uri_format = "{primary_id}/{secondary_id}/{hash}.{extension}"
diff --git a/pipelex/plugins/gateway/gateway_completions_factory.py b/pipelex/plugins/gateway/gateway_completions_factory.py
index 24a305247..ab52d754a 100644
--- a/pipelex/plugins/gateway/gateway_completions_factory.py
+++ b/pipelex/plugins/gateway/gateway_completions_factory.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import json
from typing import TYPE_CHECKING, Any, cast
import openai
@@ -133,16 +134,51 @@ class GatewayCompletionsFactory(OpenAICompletionsFactory):
case GatewayExtractProtocol.DEEPSEEK_OCR:
return cls._make_extract_output_from_response_deepseek(response=response)
+ @classmethod
+ def _extract_pages_from_choices_content(
+ cls,
+ response: GenericResponse,
+ ) -> list[dict[str, Any]] | None:
+ """Try to extract pages from choices[0].message.content (JSON string).
+
+ When Portkey proxies the response, custom top-level fields like `pages` are stripped.
+ The relay also puts pages in choices[0].message.content as a JSON string, which is
+ preserved because it's a standard OpenAI chat completion field.
+ """
+ try:
+ response_dict: dict[str, Any] = response.model_dump(serialize_as_any=True)
+ choices = response_dict.get("choices")
+ if not choices or not isinstance(choices, list) or len(choices) == 0: # pyright: ignore[reportUnknownArgumentType]
+ return None
+ first_choice = cast("dict[str, Any]", choices[0])
+ message_raw: Any = first_choice.get("message")
+ if not message_raw or not isinstance(message_raw, dict):
+ return None
+ message_dict = cast("dict[str, Any]", message_raw)
+ content: Any = message_dict.get("content")
+ if not content or not isinstance(content, str):
+ return None
+ parsed_content = json.loads(content)
+ if isinstance(parsed_content, list):
+ return cast("list[dict[str, Any]]", parsed_content)
+ return None
+ except (json.JSONDecodeError, KeyError, IndexError, TypeError, AttributeError):
+ return None
+
@classmethod
def _make_extract_output_from_response_azure(
cls,
response: GenericResponse,
) -> ExtractOutput:
- if not hasattr(response, "pages"):
- msg = "Gateway extract response does not have pages"
+ response_page_dicts: list[dict[str, Any]]
+ if hasattr(response, "pages"):
+ response_page_dicts = cast("list[dict[str, Any]]", response.pages) # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
+ elif (fallback_pages := cls._extract_pages_from_choices_content(response=response)) is not None:
+ response_page_dicts = fallback_pages
+ else:
+ msg = "Gateway extract response does not have pages (neither as top-level field nor in choices[0].message.content)"
raise GatewayExtractResponseError(msg)
try:
- response_page_dicts = cast("list[dict[str, Any]]", response.pages) # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
pages: dict[int, Page] = {}
for response_page_dict in response_page_dicts:
response_page = GatewayExtractPageAzure.model_validate(response_page_dict)
@@ -223,11 +259,15 @@ class GatewayCompletionsFactory(OpenAICompletionsFactory):
cls,
response: GenericResponse,
) -> ExtractOutput:
- if not hasattr(response, "pages"):
- msg = "Gateway extract response does not have pages"
+ response_page_dicts: list[dict[str, Any]]
+ if hasattr(response, "pages"):
+ response_page_dicts = cast("list[dict[str, Any]]", response.pages) # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
+ elif (fallback_pages := cls._extract_pages_from_choices_content(response=response)) is not None:
+ response_page_dicts = fallback_pages
+ else:
+ msg = "Gateway extract response does not have pages (neither as top-level field nor in choices[0].message.content)"
raise GatewayExtractResponseError(msg)
try:
- response_page_dicts = cast("list[dict[str, Any]]", response.pages) # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
pages: dict[int, Page] = {}
for response_page_dict in response_page_dicts:
response_page = GatewayExtractPageDeepseek.model_validate(response_page_dict)
diff --git a/pipelex/plugins/gateway/gateway_extract_worker.py b/pipelex/plugins/gateway/gateway_extract_worker.py
index 775eb40cc..791c05f9a 100644
--- a/pipelex/plugins/gateway/gateway_extract_worker.py
+++ b/pipelex/plugins/gateway/gateway_extract_worker.py
@@ -21,20 +21,6 @@ from pipelex.plugins.gateway.gateway_deck import GatewayDeck
from pipelex.plugins.gateway.gateway_factory import GatewayFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
from pipelex.tools.uri.uri_resolver import make_base64_url_from_any_uri
-from pipelex.types import StrEnum
-
-
-class DocumentKind(StrEnum):
- IMAGE = "image"
- DOCUMENT = "document"
-
- @property
- def document_tag(self) -> str:
- match self:
- case DocumentKind.IMAGE:
- return "image_url"
- case DocumentKind.DOCUMENT:
- return "document_url"
class GatewayExtractWorker(ExtractWorkerAbstract):
@@ -93,7 +79,6 @@ class GatewayExtractWorker(ExtractWorkerAbstract):
extract_output = await self._extract_base64_url(
extract_job=extract_job,
base64_url=base64_url,
- document_type=DocumentKind.IMAGE,
should_include_images=False,
)
@@ -105,7 +90,6 @@ class GatewayExtractWorker(ExtractWorkerAbstract):
extract_output = await self._extract_base64_url(
extract_job=extract_job,
base64_url=base64_url,
- document_type=DocumentKind.DOCUMENT,
should_include_images=should_include_images,
)
else:
@@ -117,15 +101,11 @@ class GatewayExtractWorker(ExtractWorkerAbstract):
self,
extract_job: ExtractJob,
base64_url: str,
- document_type: DocumentKind,
should_include_images: bool = False,
) -> ExtractOutput:
config_id = GatewayDeck.get_config_id(headers=self.inference_model.extra_headers or {})
log.dev(f"Extracting using config '{config_id}' with should_include_images: {should_include_images}")
- doc_tag = document_type.document_tag
- base_body: dict[str, Any] = {}
- base_body[document_type.value] = {"type": doc_tag, doc_tag: base64_url}
attempt_number = 0
response: GenericResponse | None = None
retryer = self._make_retryer()
@@ -133,12 +113,20 @@ class GatewayExtractWorker(ExtractWorkerAbstract):
extra_headers, extra_body = GatewayFactory.make_extras(
inference_model=self.inference_model, inference_job=extract_job, output_desc=InferenceOutputType.PAGES
)
- extra_body.update(base_body)
+ # Encode document/image as an image_url content part inside messages.
+ # Portkey forwards messages but strips custom top-level fields like document/image.
+ if extra_body.get("messages"):
+ first_msg = extra_body["messages"][0]
+ text_content = first_msg["content"]
+ first_msg["content"] = [
+ {"type": "text", "text": text_content},
+ {"type": "image_url", "image_url": {"url": base64_url}},
+ ]
async for attempt in retryer:
with attempt:
attempt_number += 1
response = await self.portkey_client.with_options(config=config_id).post( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- "/",
+ "/chat/completions",
model=self.inference_model.model_id,
headers=extra_headers,
**extra_body,
diff --git a/pipelex/plugins/gateway/gateway_img_gen_worker.py b/pipelex/plugins/gateway/gateway_img_gen_worker.py
index a9cae376e..6abf53153 100644
--- a/pipelex/plugins/gateway/gateway_img_gen_worker.py
+++ b/pipelex/plugins/gateway/gateway_img_gen_worker.py
@@ -16,6 +16,7 @@ from pipelex.cogt.image.generated_image import GeneratedImageRawDetails
from pipelex.cogt.image.image_size import ImageSize
from pipelex.cogt.img_gen.img_gen_args_factory import ImgGenArgsFactory
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
from pipelex.plugins.fal.fal_poller import FalPoller
from pipelex.plugins.gateway.gateway_deck import GatewayDeck
from pipelex.plugins.gateway.gateway_factory import GatewayFactory
@@ -90,9 +91,19 @@ class GatewayImgGenWorker(ImgGenWorkerAbstract):
msg = "Response is not of type GenericResponse"
raise TypeError(msg)
- # Giv en that different backends and different models can be used with this worker, we must interpret the response,
- # so we do it according to the shape we detext
+ # Given that different backends and different models can be used with this worker, we must interpret the response,
+ # so we do it according to the shape we detect
response_dict: dict[str, Any] = response.model_dump(serialize_as_any=True)
+
+ # Extract usage tokens if available
+ if (usage_dict := response_dict.get("usage")) and (img_gen_tokens_usage := img_gen_job.job_report.img_gen_tokens_usage):
+ nb_tokens: NbTokensByCategoryDict = {}
+ if input_tokens := usage_dict.get("prompt_tokens") or usage_dict.get("input_tokens"):
+ nb_tokens[TokenCategory.INPUT] = input_tokens
+ if output_tokens := usage_dict.get("completion_tokens") or usage_dict.get("output_tokens"):
+ nb_tokens[TokenCategory.OUTPUT] = output_tokens
+ img_gen_tokens_usage.nb_tokens_by_category = nb_tokens
+
generated_images: list[GeneratedImageRawDetails] = []
if images := response_dict.get("data"):
# Azure-shaped responses, model is either OpenAI's GPT Image or Black Forest Labs' Flux 2 Pro
diff --git a/pipelex/plugins/google/google_img_gen_worker.py b/pipelex/plugins/google/google_img_gen_worker.py
index b527ee0ac..67d6f4919 100644
--- a/pipelex/plugins/google/google_img_gen_worker.py
+++ b/pipelex/plugins/google/google_img_gen_worker.py
@@ -13,6 +13,7 @@ from pipelex.cogt.image.image_size import ImageSize
from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.plugins.google.google_factory import GoogleFactory
from pipelex.plugins.google.google_img_gen_factory import GoogleImgGenFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
@@ -117,6 +118,13 @@ class GoogleImgGenWorker(ImgGenWorkerAbstract):
if not usage_metadata:
log.warning("No usage metadata returned from Google")
+ if usage_metadata and (img_gen_tokens_usage := img_gen_job.job_report.img_gen_tokens_usage):
+ request_tokens = GoogleFactory.extract_token_usage(usage_metadata)
+ for token_category, nb_tokens in request_tokens.items():
+ img_gen_tokens_usage.nb_tokens_by_category[token_category] = (
+ img_gen_tokens_usage.nb_tokens_by_category.get(token_category, 0) + nb_tokens
+ )
+
# Extract image from response
if not response.candidates:
msg = f"No candidates returned from model: {self.inference_model.desc}"
diff --git a/pipelex/plugins/openai/openai_completions_img_gen_worker.py b/pipelex/plugins/openai/openai_completions_img_gen_worker.py
index 53c4801b9..431bb7c85 100644
--- a/pipelex/plugins/openai/openai_completions_img_gen_worker.py
+++ b/pipelex/plugins/openai/openai_completions_img_gen_worker.py
@@ -129,6 +129,10 @@ class OpenAICompletionsImgGenWorker(ImgGenWorkerAbstract):
if not base64_str and not actual_url:
msg = f"ImgGenCompletions response has no image. Model: {self.inference_model.desc}"
raise ImgGenGenerationError(msg)
+
+ if (img_gen_tokens_usage := img_gen_job.job_report.img_gen_tokens_usage) and (usage := response.usage):
+ img_gen_tokens_usage.nb_tokens_by_category = self.openai_completions_factory.make_nb_tokens_by_category(usage=usage)
+
# Size is None because the API doesn't return it. We now support various aspect ratios,
# but detecting the size here (e.g., via Pillow) is left to downstream consumers if needed.
return GeneratedImageRawDetails(
diff --git a/pipelex/plugins/openai/openai_img_gen_worker.py b/pipelex/plugins/openai/openai_img_gen_worker.py
index 2ed15d256..4cd111239 100644
--- a/pipelex/plugins/openai/openai_img_gen_worker.py
+++ b/pipelex/plugins/openai/openai_img_gen_worker.py
@@ -10,6 +10,7 @@ from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
from pipelex.cogt.img_gen.img_gen_job_components import Quality
from pipelex.cogt.img_gen.img_gen_worker_abstract import ImgGenWorkerAbstract
from pipelex.cogt.model_backends.model_spec import InferenceModelSpec
+from pipelex.cogt.usage.token_category import NbTokensByCategoryDict, TokenCategory
from pipelex.plugins.openai.openai_img_gen_factory import OpenAIImgGenFactory
from pipelex.reporting.reporting_protocol import ReportingProtocol
@@ -88,6 +89,13 @@ class OpenAIImgGenWorker(ImgGenWorkerAbstract):
msg = "No usage received from OpenAI"
raise ImgGenGenerationError(msg)
+ if img_gen_tokens_usage := img_gen_job.job_report.img_gen_tokens_usage:
+ nb_tokens: NbTokensByCategoryDict = {
+ TokenCategory.INPUT: usage.input_tokens,
+ TokenCategory.OUTPUT: usage.output_tokens,
+ }
+ img_gen_tokens_usage.nb_tokens_by_category = nb_tokens
+
generated_images: list[GeneratedImageRawDetails] = []
for image_data in images_response.data:
base64_str = image_data.b64_json
diff --git a/pipelex/reporting/reporting_manager.py b/pipelex/reporting/reporting_manager.py
index 535a03a8c..44addf246 100644
--- a/pipelex/reporting/reporting_manager.py
+++ b/pipelex/reporting/reporting_manager.py
@@ -5,6 +5,10 @@ from typing_extensions import override
from pipelex import log
from pipelex.cogt.exceptions import ReportingManagerError
+from pipelex.cogt.extract.extract_job import ExtractJob
+from pipelex.cogt.extract.extract_report import ExtractTokensUsage
+from pipelex.cogt.img_gen.img_gen_job import ImgGenJob
+from pipelex.cogt.img_gen.img_gen_report import ImgGenTokensUsage
from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_report import LLMTokensUsage
@@ -18,17 +22,18 @@ from pipelex.tools.typing.pydantic_utils import empty_list_factory_of
if TYPE_CHECKING:
from pathlib import Path
-LLMUsageRegistryRoot = list[LLMTokensUsage]
+TokensUsage = LLMTokensUsage | ImgGenTokensUsage | ExtractTokensUsage
+UsageRegistryRoot = list[TokensUsage]
-class UsageRegistry(RootModel[LLMUsageRegistryRoot]):
- root: LLMUsageRegistryRoot = Field(default_factory=empty_list_factory_of(LLMTokensUsage))
+class UsageRegistry(RootModel[UsageRegistryRoot]):
+ root: UsageRegistryRoot = Field(default_factory=empty_list_factory_of(LLMTokensUsage))
- def get_current_tokens_usage(self) -> LLMUsageRegistryRoot:
+ def get_current_tokens_usage(self) -> UsageRegistryRoot:
return self.root
- def add_tokens_usage(self, llm_tokens_usage: LLMTokensUsage):
- self.root.append(llm_tokens_usage)
+ def add_tokens_usage(self, tokens_usage: TokensUsage):
+ self.root.append(tokens_usage)
class ReportingManager(ReportingProtocol):
@@ -70,9 +75,37 @@ class ReportingManager(ReportingProtocol):
self._get_registry(pipeline_run_id).add_tokens_usage(llm_tokens_usage)
if self._reporting_config.is_log_costs_to_console:
- llm_token_cost_report = CostRegistry.complete_cost_report(llm_tokens_usage=llm_tokens_usage)
+ llm_token_cost_report = CostRegistry.complete_cost_report(tokens_usage=llm_tokens_usage)
log.verbose(llm_token_cost_report, title="Token Cost report")
+ def _report_img_gen_job(self, img_gen_job: ImgGenJob):
+ img_gen_tokens_usage = img_gen_job.job_report.img_gen_tokens_usage
+
+ if not img_gen_tokens_usage:
+ log.warning("ImgGen job has no img_gen_tokens_usage")
+ return
+
+ pipeline_run_id = img_gen_job.job_metadata.pipeline_run_id
+ self._get_registry(pipeline_run_id).add_tokens_usage(img_gen_tokens_usage)
+
+ if self._reporting_config.is_log_costs_to_console:
+ img_gen_token_cost_report = CostRegistry.complete_cost_report(tokens_usage=img_gen_tokens_usage)
+ log.verbose(img_gen_token_cost_report, title="Token Cost report")
+
+ def _report_extract_job(self, extract_job: ExtractJob):
+ extract_tokens_usage = extract_job.job_report.extract_tokens_usage
+
+ if not extract_tokens_usage:
+ log.warning("Extract job has no extract_tokens_usage")
+ return
+
+ pipeline_run_id = extract_job.job_metadata.pipeline_run_id
+ self._get_registry(pipeline_run_id).add_tokens_usage(extract_tokens_usage)
+
+ if self._reporting_config.is_log_costs_to_console:
+ extract_token_cost_report = CostRegistry.complete_cost_report(tokens_usage=extract_tokens_usage)
+ log.verbose(extract_token_cost_report, title="Token Cost report")
+
############################################################
# ReportingProtocol
############################################################
@@ -87,12 +120,14 @@ class ReportingManager(ReportingProtocol):
@override
def report_inference_job(self, inference_job: InferenceJobAbstract):
log.verbose(f"Inference job '{inference_job.job_metadata.unit_job_id}' completed in {inference_job.job_metadata.duration:.2f} seconds")
- if not isinstance(inference_job, LLMJob):
- # ReportingManager does not support reporting for other types of inference jobs yet
- # TODO: add support for other types of inference jobs
- return
- llm_job: LLMJob = inference_job
- self._report_llm_job(llm_job=llm_job)
+ if isinstance(inference_job, LLMJob):
+ self._report_llm_job(llm_job=inference_job)
+ elif isinstance(inference_job, ImgGenJob):
+ self._report_img_gen_job(img_gen_job=inference_job)
+ elif isinstance(inference_job, ExtractJob):
+ self._report_extract_job(extract_job=inference_job)
+ else:
+ log.warning(f"ReportingManager does not support reporting for inference job type: {type(inference_job).__name__}")
@override
def generate_report(self, pipeline_run_id: str | None = None):
@@ -114,7 +149,7 @@ class ReportingManager(ReportingProtocol):
for run_id, registry in registries_to_process.items():
CostRegistry.generate_report(
pipeline_run_id=run_id,
- llm_tokens_usages=registry.get_current_tokens_usage(),
+ tokens_usages=registry.get_current_tokens_usage(),
unit_scale=self._reporting_config.cost_report_unit_scale,
cost_report_file_path=cost_report_file_path,
)
diff --git a/pipelex/system/pipelex_service/pipelex_details.py b/pipelex/system/pipelex_service/pipelex_details.py
index 3fdfc4cf5..973c84f28 100644
--- a/pipelex/system/pipelex_service/pipelex_details.py
+++ b/pipelex/system/pipelex_service/pipelex_details.py
@@ -2,7 +2,7 @@ from pipelex.tools.misc.hash_utils import hash_sha256
class PipelexDetails:
- REMOTE_CONFIG_URL = "https://pipelex-config.s3.eu-west-3.amazonaws.com/pipelex_remote_config_04.json"
+ REMOTE_CONFIG_URL = "https://pipelex-config.s3.eu-west-3.amazonaws.com/pipelex_remote_config_05.json"
PIPELEX_GATEWAY_API_KEY_VAR = "PIPELEX_GATEWAY_API_KEY"
@classmethod
diff --git a/tests/integration/pipelex/cogt/test_extract.py b/tests/integration/pipelex/cogt/test_extract.py
index a970cc045..0e8969f66 100644
--- a/tests/integration/pipelex/cogt/test_extract.py
+++ b/tests/integration/pipelex/cogt/test_extract.py
@@ -5,7 +5,7 @@ from pipelex.cogt.content_generation.generated_content_factory import GeneratedC
from pipelex.cogt.extract.extract_input import ExtractInput
from pipelex.cogt.extract.extract_job_components import ExtractJobParams
from pipelex.cogt.extract.extract_job_factory import ExtractJobFactory
-from pipelex.hub import get_extract_worker
+from pipelex.hub import get_extract_worker, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from tests.cases import DocumentTestCases, ImageTestCases
from tests.integration.pipelex.fixtures.model_combo import ModelCombo
@@ -48,6 +48,7 @@ class TestExtract:
assert page_contents
for page_index, page_content in enumerate(page_contents):
pretty_print(page_content, title=f"Page {page_index}")
+ get_report_delegate().generate_report()
@pytest.mark.parametrize("url", DocumentTestCases.DOCUMENT_URLS)
async def test_extract_pdf_url(
diff --git a/tests/integration/pipelex/cogt/test_img_gen.py b/tests/integration/pipelex/cogt/test_img_gen.py
index 6a52cb71b..9924fb6b0 100644
--- a/tests/integration/pipelex/cogt/test_img_gen.py
+++ b/tests/integration/pipelex/cogt/test_img_gen.py
@@ -4,7 +4,7 @@ from pipelex import pretty_print, pretty_print_url
from pipelex.cogt.content_generation.generated_content_factory import GeneratedContentFactory
from pipelex.cogt.img_gen.img_gen_job_components import AspectRatio, Background, ImgGenJobParams
from pipelex.cogt.img_gen.img_gen_job_factory import ImgGenJobFactory
-from pipelex.hub import get_img_gen_worker
+from pipelex.hub import get_img_gen_worker, get_report_delegate
from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.misc.image_utils import ImageFormat
from tests.integration.pipelex.fixtures.model_combo import ModelCombo
@@ -50,6 +50,7 @@ class TestImageGeneration:
pretty_print(image_content, title=f"Image content for topic '{topic}'")
assert image_content.public_url is not None
pretty_print_url(image_content.public_url, title=f"Image URL for topic '{topic}'")
+ get_report_delegate().generate_report()
@pytest.mark.parametrize(("topic", "positive_text", "negative_text"), ImageGenTestCases.IMAGE_GEN_PROMPT_CONTENTS)
async def test_img_gen_single_transparent(
diff --git a/tests/integration/pipelex/fixtures/extract_fixtures.py b/tests/integration/pipelex/fixtures/extract_fixtures.py
index f2115b270..38469edc0 100644
--- a/tests/integration/pipelex/fixtures/extract_fixtures.py
+++ b/tests/integration/pipelex/fixtures/extract_fixtures.py
@@ -103,7 +103,7 @@ def extract_choice_for_image(request: pytest.FixtureRequest) -> str:
max_nb_images=None,
should_caption_images=False,
should_include_page_views=False,
- page_views_dpi=72,
+ page_views_dpi=None,
image_min_size=None,
),
# max_nb_images=10: Limit to 10 images
diff --git a/tests/unit/pipelex/cogt/usage/test_cost_registry.py b/tests/unit/pipelex/cogt/usage/test_cost_registry.py
index bbc5c954f..b0a0dbac4 100644
--- a/tests/unit/pipelex/cogt/usage/test_cost_registry.py
+++ b/tests/unit/pipelex/cogt/usage/test_cost_registry.py
@@ -32,7 +32,7 @@ class TestCostRegistry:
},
)
- cost_report = CostRegistry.complete_cost_report(llm_tokens_usage=llm_tokens_usage)
+ cost_report = CostRegistry.complete_cost_report(tokens_usage=llm_tokens_usage)
cost_registry = CostRegistry(root=[cost_report])
# Convert to records
@@ -234,7 +234,7 @@ class TestCostRegistry:
csv_file = tmp_path / "aggregation_test.csv"
CostRegistry.generate_report(
pipeline_run_id="test-pipeline",
- llm_tokens_usages=llm_tokens_usages,
+ tokens_usages=llm_tokens_usages,
unit_scale=1.0,
cost_report_file_path=csv_file,
)
@@ -284,7 +284,7 @@ class TestCostRegistry:
# Test with non-untitled pipeline
CostRegistry.generate_report(
pipeline_run_id="test-pipeline",
- llm_tokens_usages=[],
+ tokens_usages=[],
unit_scale=1.0,
cost_report_file_path=None,
)
@@ -293,7 +293,7 @@ class TestCostRegistry:
# Test with untitled pipeline
CostRegistry.generate_report(
pipeline_run_id="untitled",
- llm_tokens_usages=[],
+ tokens_usages=[],
unit_scale=1.0,
cost_report_file_path=None,
)
@@ -325,7 +325,7 @@ class TestCostRegistry:
csv_file = tmp_path / "cost_report.csv"
CostRegistry.generate_report(
pipeline_run_id="test-pipeline",
- llm_tokens_usages=[llm_tokens_usage],
+ tokens_usages=[llm_tokens_usage],
unit_scale=1.0,
cost_report_file_path=csv_file,
)
@@ -388,7 +388,7 @@ class TestCostRegistry:
# Generate report
CostRegistry.generate_report(
pipeline_run_id="test-pipeline",
- llm_tokens_usages=[llm_tokens_usage],
+ tokens_usages=[llm_tokens_usage],
unit_scale=unit_scale,
cost_report_file_path=None,
)
@@ -412,11 +412,11 @@ class TestCostRegistry:
assert data_row_call is not None, "Could not find data row call"
- # Extract the cost value from the formatted string (6th argument, index 6)
- # Columns: Model, Input Cached tokens, Input Non Cached tokens, Input Joined tokens, Output tokens,
- # Input Cached Cost (5), Input Non Cached Cost (6), Input Joined Cost (7), Output Cost (8), Total Cost (9)
+ # Extract the cost value from the formatted string (7th argument, index 7)
+ # Columns: Model, Type, Input Cached tokens, Input Non Cached tokens, Input Joined tokens, Output tokens,
+ # Input Cached Cost (6), Input Non Cached Cost (7), Input Joined Cost (8), Output Cost (9), Total Cost (10)
args: tuple[Any, ...] = data_row_call[0]
- cost_str: str = str(args[6]) # Input Non Cached Cost column
+ cost_str: str = str(args[7]) # Input Non Cached Cost column
# Extract the numeric value and verify it matches expected scaled cost
actual_cost = float(cost_str)