Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Feature/pipelex get instance (#85)
Louis Choquelf0f9e5203fe0
diff --git a/README.md b/README.mdindex 714da7918..32beb7973 100644--- a/README.md+++ b/README.md@@ -1,7 +1,7 @@<div align="center"><a href="https://www.pipelex.com/"><img src="https://raw.githubusercontent.com/Pipelex/pipelex/main/.github/assets/logo.png" alt="Pipelex Logo" width="400" style="max-width: 100%; height: auto;"></a>- <h2 align="center">Lean-code language for repeatable workflows</h2>+ <h2 align="center">Open-source language for repeatable AI workflows</h2><p align="center">Pipelex is based on a simple declarative language that lets you define repeatable, structured, composable AI workflows.</p><div>diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.pyindex 8fb24e62a..aef295bb3 100644--- a/pipelex/cogt/config_cogt.py+++ b/pipelex/cogt/config_cogt.py@@ -12,6 +12,7 @@ from pipelex.tools.config.models import ConfigModelclass OcrConfig(ConfigModel):ocr_handles: List[str]page_output_text_file_name: str+ default_page_views_dpi: intclass ImggConfig(ConfigModel):diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex 89f3a2398..4e13e72a9 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -136,6 +136,7 @@ class ContentGenerator(ContentGeneratorProtocol):object_class: Type[BaseModelTypeVar],llm_setting_for_object_list: LLMSetting,llm_prompt_for_object_list: LLMPrompt,+ nb_items: Optional[int] = None,wfid: Optional[str] = None,) -> List[BaseModelTypeVar]:llm_assignment_for_object = LLMAssignment(@@ -161,6 +162,7 @@ class ContentGenerator(ContentGeneratorProtocol):llm_setting_for_object_list: LLMSetting,llm_prompt_for_text: LLMPrompt,llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,+ nb_items: Optional[int] = None,wfid: Optional[str] = None,) -> List[BaseModelTypeVar]:llm_assignment_for_text = LLMAssignment(diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.pyindex e295359f3..538e8b6f5 100644--- a/pipelex/cogt/content_generation/content_generator_dry.py+++ b/pipelex/cogt/content_generation/content_generator_dry.py@@ -63,6 +63,7 @@ class ContentGeneratorDry(ContentGeneratorProtocol):class ObjectFactory(ModelFactory[object_class]): # type: ignore__model__ = object_class+ __use_examples__ = Trueobj = ObjectFactory.build()return obj@@ -96,24 +97,22 @@ class ContentGeneratorDry(ContentGeneratorProtocol):object_class: Type[BaseModelTypeVar],llm_setting_for_object_list: LLMSetting,llm_prompt_for_object_list: LLMPrompt,+ nb_items: Optional[int] = None,wfid: Optional[str] = None,) -> List[BaseModelTypeVar]:func_name = "make_object_list_direct"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")- object_1 = await self.make_object_direct(- job_metadata=job_metadata,- object_class=object_class,- llm_setting_for_object=llm_setting_for_object_list,- llm_prompt_for_object=llm_prompt_for_object_list,- )- object_2 = await self.make_object_direct(- job_metadata=job_metadata,- object_class=object_class,- llm_setting_for_object=llm_setting_for_object_list,- llm_prompt_for_object=llm_prompt_for_object_list,- )- two_objects = [object_1, object_2]- return two_objects+ nb_list_items = nb_items or get_config().pipelex.dry_run_config.nb_list_items+ objects = [+ await self.make_object_direct(+ job_metadata=job_metadata,+ object_class=object_class,+ llm_setting_for_object=llm_setting_for_object_list,+ llm_prompt_for_object=llm_prompt_for_object_list,+ )+ for _ in range(nb_list_items)+ ]+ return objects@override@update_job_metadata@@ -125,6 +124,7 @@ class ContentGeneratorDry(ContentGeneratorProtocol):llm_setting_for_object_list: LLMSetting,llm_prompt_for_text: LLMPrompt,llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,+ nb_items: Optional[int] = None,wfid: Optional[str] = None,) -> List[BaseModelTypeVar]:func_name = "make_text_then_object_list"@@ -134,6 +134,7 @@ class ContentGeneratorDry(ContentGeneratorProtocol):object_class=object_class,llm_setting_for_object_list=llm_setting_for_object_list,llm_prompt_for_object_list=llm_prompt_for_text,+ nb_items=nb_items,)@override@@ -149,8 +150,10 @@ class ContentGeneratorDry(ContentGeneratorProtocol):) -> GeneratedImage:func_name = "make_single_image"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")+ image_urls = get_config().pipelex.dry_run_config.image_urls+ image_url = image_urls[0]generated_image = GeneratedImage(- url="https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",+ url=image_url,width=1536,height=2752,)@@ -170,17 +173,14 @@ class ContentGeneratorDry(ContentGeneratorProtocol):) -> List[GeneratedImage]:func_name = "make_image_list"log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")+ image_urls = get_config().pipelex.dry_run_config.image_urlsgenerated_image_list = [GeneratedImage(- url="https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",+ url=image_urls[image_index % len(image_urls)],width=1536,height=2752,- ),- GeneratedImage(- url="https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",- width=1024,- height=1536,- ),+ )+ for image_index in range(nb_images)]return generated_image_list@@ -225,25 +225,18 @@ class ContentGeneratorDry(ContentGeneratorProtocol):pages={1: ocr_image_as_page},)else:- ocr_page_1 = Page(- text="DRY RUN: OCR text",- extracted_images=[],- page_view=ExtractedImageFromPage(- image_id="page_view_1",- base_64="",- caption="DRY RUN: OCR text",- ),- )- ocr_page_2 = Page(- text="DRY RUN: OCR text",- extracted_images=[],- page_view=ExtractedImageFromPage(- image_id="page_view_2",- base_64="",- caption="DRY RUN: OCR text",- ),- )- ocr_output = OcrOutput(- pages={1: ocr_page_1, 2: ocr_page_2},- )+ nb_pages = get_config().pipelex.dry_run_config.nb_ocr_pages+ pages = {+ page_index: Page(+ text="DRY RUN: OCR text",+ extracted_images=[],+ page_view=ExtractedImageFromPage(+ image_id=f"page_view_{page_index}",+ base_64="",+ caption="DRY RUN: OCR text",+ ),+ )+ for page_index in range(1, nb_pages + 1)+ }+ ocr_output = OcrOutput(pages=pages)return ocr_outputdiff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.pyindex 034badbd7..52ce35ec9 100644--- a/pipelex/cogt/content_generation/content_generator_protocol.py+++ b/pipelex/cogt/content_generation/content_generator_protocol.py@@ -78,6 +78,7 @@ class ContentGeneratorProtocol(Protocol):object_class: Type[BaseModelTypeVar],llm_setting_for_object_list: LLMSetting,llm_prompt_for_object_list: LLMPrompt,+ nb_items: Optional[int] = None,wfid: Optional[str] = None,) -> List[BaseModelTypeVar]: ...@@ -89,6 +90,7 @@ class ContentGeneratorProtocol(Protocol):llm_setting_for_object_list: LLMSetting,llm_prompt_for_text: LLMPrompt,llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,+ nb_items: Optional[int] = None,wfid: Optional[str] = None,) -> List[BaseModelTypeVar]: ...diff --git a/pipelex/cogt/ocr/ocr_job_components.py b/pipelex/cogt/ocr/ocr_job_components.pyindex 7a06cfdd3..f74f322e8 100644--- a/pipelex/cogt/ocr/ocr_job_components.py+++ b/pipelex/cogt/ocr/ocr_job_components.py@@ -1,3 +1,5 @@+from typing import Optional+from pydantic import BaseModelfrom pipelex.tools.config.models import ConfigModel@@ -7,7 +9,7 @@ class OcrJobParams(BaseModel):should_include_images: boolshould_caption_images: boolshould_include_page_views: bool- page_views_dpi: int+ page_views_dpi: Optional[int] = None@classmethoddef make_default_ocr_job_params(cls) -> "OcrJobParams":@@ -15,7 +17,7 @@ class OcrJobParams(BaseModel):should_caption_images=False,should_include_page_views=False,should_include_images=True,- page_views_dpi=300,+ page_views_dpi=None,)diff --git a/pipelex/config.py b/pipelex/config.pyindex b2373df4a..87005e265 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -5,7 +5,7 @@ from pydantic import Field, field_validatorfrom pipelex.cogt.config_cogt import Cogtfrom pipelex.cogt.llm.llm_models.llm_prompting_target import LLMPromptingTarget-from pipelex.exceptions import PipelexError, StaticValidationErrorType+from pipelex.exceptions import PipelexConfigError, StaticValidationErrorTypefrom pipelex.hub import get_required_configfrom pipelex.libraries.library_config import LibraryConfigfrom pipelex.pipeline.track.tracker_config import TrackerConfig@@ -40,10 +40,6 @@ class StaticValidationConfig(ConfigModel):return the_dict-class PipelexConfigError(PipelexError):- pass--class PipeRunConfig(ConfigModel):pipe_stack_limit: int@@ -51,6 +47,15 @@ class PipeRunConfig(ConfigModel):class DryRunConfig(ConfigModel):apply_to_jinja2_rendering: booltext_gen_truncate_length: int+ nb_list_items: int+ nb_ocr_pages: int+ image_urls: List[str]++ @field_validator("image_urls", mode="before")+ def validate_image_urls(cls, value: List[str]) -> List[str]:+ if not value:+ raise PipelexConfigError("dry_run_config.image_urls must be a non-empty list")+ return valueclass GenericTemplateNames(ConfigModel):diff --git a/pipelex/core/domain_library.py b/pipelex/core/domain_library.pyindex 8225ec972..f2793042b 100644--- a/pipelex/core/domain_library.py+++ b/pipelex/core/domain_library.py@@ -3,7 +3,7 @@ from typing import Dict, List, Optionalfrom pydantic import Field, RootModelfrom typing_extensions import override-from pipelex import log+from pipelex import log, pretty_printfrom pipelex.core.domain import Domainfrom pipelex.core.domain_provider_abstract import DomainProviderAbstractfrom pipelex.exceptions import DomainLibraryError@@ -21,11 +21,13 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):def reset(self):self.root = {}- def add_new_domain(self, domain: Domain):+ def add_domain_details(self, domain: Domain):domain_code = domain.code- if domain_code in self.root:- raise DomainLibraryError(f"Domain '{domain_code}' already exists in the library")- self.root[domain_code] = domain+ if existing_domain := self.root.get(domain_code):+ # merge the new domain with the existing one+ self.root[domain_code] = existing_domain.model_copy(update=domain.model_dump())+ else:+ self.root[domain_code] = domaindef print_all(self):log.dev("-" * 80)@@ -58,6 +60,7 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):def get_required_domain(self, domain_code: str) -> Domain:the_domain = self.get_domain(domain_code=domain_code)if not the_domain:+ pretty_print(self.root, title="DomainLibrary")raise DomainLibraryError(f"Domain '{domain_code}' not found. Check for typos and make sure it is declared in a pipeline library.")return the_domaindiff --git a/pipelex/core/pipe_run_params.py b/pipelex/core/pipe_run_params.pyindex 20df46d41..1049503e6 100644--- a/pipelex/core/pipe_run_params.py+++ b/pipelex/core/pipe_run_params.py@@ -18,6 +18,8 @@ class PipeRunMode(StrEnum):DRY = "dry"+FORCE_DRY_RUN_MODE_ENV_KEY = "PIPELEX_FORCE_DRY_RUN_MODE"+PipeOutputMultiplicity = Union[bool, int]diff --git a/pipelex/core/stuff.py b/pipelex/core/stuff.pyindex 4d4e211d8..48e0fbeb9 100644--- a/pipelex/core/stuff.py+++ b/pipelex/core/stuff.py@@ -20,13 +20,14 @@ from pipelex.core.stuff_content import ()from pipelex.exceptions import StuffErrorfrom pipelex.tools.misc.string_utils import pascal_case_to_snake_case+from pipelex.tools.typing.pydantic_utils import CustomBaseModelclass StuffCreationRecord(BaseModel):pass-class Stuff(BaseModel):+class Stuff(CustomBaseModel):model_config = ConfigDict(extra="ignore", strict=True)stuff_code: strdiff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.pyindex f527f9f75..eaa7fce2d 100644--- a/pipelex/libraries/library_manager.py+++ b/pipelex/libraries/library_manager.py@@ -133,9 +133,6 @@ class LibraryManager:"Just write 'domain = \"my_domain\"' at the top of the file.")domain_definition = library_dict.get("definition")- if domain_definition is None:- # we skip the domain without definition, it must be defined one and only one time in the domain library- continuesystem_prompt = library_dict.get("system_prompt")system_prompt_to_structure = library_dict.get("system_prompt_to_structure")prompt_template_to_structure = library_dict.get("prompt_template_to_structure")@@ -146,7 +143,7 @@ class LibraryManager:system_prompt_to_structure=system_prompt_to_structure,prompt_template_to_structure=prompt_template_to_structure,)- self.domain_library.add_new_domain(domain=domain)+ self.domain_library.add_domain_details(domain=domain)# Second pass: load all conceptsfor toml_path in toml_file_paths:diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.pyindex 1a25c16da..62b510e67 100644--- a/pipelex/pipe_operators/pipe_llm.py+++ b/pipelex/pipe_operators/pipe_llm.py@@ -412,6 +412,7 @@ class PipeLLM(PipeOperator):llm_setting_main=self.llm_setting_main,llm_prompt_factory_for_object_list=llm_prompt_2_factory,llm_setting_for_object_list=self.llm_setting_for_object_list,+ nb_items=fixed_nb_output,wfid=task_desc,)else:@@ -423,6 +424,7 @@ class PipeLLM(PipeOperator):object_class=content_class,llm_prompt_for_object_list=llm_prompt_1,llm_setting_for_object_list=self.llm_setting_for_object_list_direct,+ nb_items=fixed_nb_output,wfid=task_desc,)diff --git a/pipelex/pipe_operators/pipe_ocr_factory.py b/pipelex/pipe_operators/pipe_ocr_factory.pyindex 6d4e7290c..19ba7a48b 100644--- a/pipelex/pipe_operators/pipe_ocr_factory.py+++ b/pipelex/pipe_operators/pipe_ocr_factory.py@@ -4,6 +4,7 @@ from typing_extensions import overridefrom pipelex.cogt.ocr.ocr_engine_factory import OcrEngineFactory, OcrPlatformfrom pipelex.cogt.ocr.ocr_handle import OcrHandle+from pipelex.config import get_configfrom pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocolfrom pipelex.core.pipe_input_spec import PipeInputSpecfrom pipelex.pipe_operators.pipe_ocr import PipeOcr@@ -15,7 +16,7 @@ class PipeOcrBlueprint(PipeBlueprint):page_images: bool = Falsepage_image_captions: bool = Falsepage_views: bool = False- page_views_dpi: int = 300+ page_views_dpi: Optional[int] = Noneclass PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):@@ -42,7 +43,7 @@ class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):should_include_images=pipe_blueprint.page_images,should_caption_images=pipe_blueprint.page_image_captions,should_include_page_views=pipe_blueprint.page_views,- page_views_dpi=pipe_blueprint.page_views_dpi,+ page_views_dpi=pipe_blueprint.page_views_dpi or get_config().cogt.ocr_config.default_page_views_dpi,)@classmethoddiff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 50a3d5a82..a0c75361a 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -243,3 +243,9 @@ class Pipelex:pipelex_instance.finish_setup()log.info(f"Pipelex {PACKAGE_VERSION} initialized.")return pipelex_instance++ @classmethod+ def get_instance(cls) -> Self:+ if cls._pipelex_instance is None:+ raise RuntimeError("Pipelex is not initialized")+ return cls._pipelex_instancediff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 79431a752..345f6f142 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -128,6 +128,7 @@ seed = "auto"[cogt.ocr_config]ocr_handles = ["mistral/mistral-ocr-latest"]page_output_text_file_name = "page_text.md"+default_page_views_dpi = 72##################################################################################################### Plugins config@@ -245,3 +246,10 @@ pipe_stack_limit = 20[pipelex.dry_run_config]apply_to_jinja2_rendering = falsetext_gen_truncate_length = 256+nb_list_items = 3+nb_ocr_pages = 4+image_urls = [+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",+ ]+diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.pyindex 06db5bded..b83b9dfd0 100644--- a/pipelex/pipeline/execute.py+++ b/pipelex/pipeline/execute.py@@ -2,12 +2,13 @@ from typing import Optional, Tuplefrom pipelex import pretty_printfrom pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode+from pipelex.core.pipe_run_params import FORCE_DRY_RUN_MODE_ENV_KEY, PipeOutputMultiplicity, PipeRunModefrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipefrom pipelex.pipe_works.pipe_job_factory import PipeJobFactoryfrom pipelex.pipeline.job_metadata import JobMetadata+from pipelex.tools.environment import get_optional_envasync def execute_pipeline(@@ -16,7 +17,7 @@ async def execute_pipeline(output_name: Optional[str] = None,output_multiplicity: Optional[PipeOutputMultiplicity] = None,dynamic_output_concept_code: Optional[str] = None,- pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,+ pipe_run_mode: Optional[PipeRunMode] = None,) -> Tuple[PipeOutput, str]:"""Execute a pipeline and wait for its completion.@@ -37,13 +38,22 @@ async def execute_pipeline(dynamic_output_concept_code:Override the dynamic output concept code.pipe_run_mode:- Pipe run mode: ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``.+ Pipe run mode: if specified, it must be ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``.+ If not specified, the pipe run mode is inferred from the environment variable+ ``PIPELEX_FORCE_DRY_RUN_MODE``. If the environment variable is not set,+ the pipe run mode is ``PipeRunMode.LIVE``.Returns-------Tuple[PipeOutput, str]A tuple containing the pipe output and the pipeline run ID."""+ if pipe_run_mode is None:+ if run_mode_from_env := get_optional_env(key=FORCE_DRY_RUN_MODE_ENV_KEY):+ pipe_run_mode = PipeRunMode(run_mode_from_env)+ else:+ pipe_run_mode = PipeRunMode.LIVE+pipeline = get_pipeline_manager().add_new_pipeline()pipeline_run_id = pipeline.pipeline_run_idget_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)diff --git a/pipelex/tools/log/log_dispatch.py b/pipelex/tools/log/log_dispatch.pyindex db50ec652..f5031925e 100644--- a/pipelex/tools/log/log_dispatch.py+++ b/pipelex/tools/log/log_dispatch.py@@ -209,7 +209,11 @@ class LogDispatch:self._log_to_console(message=message, severity=severity)elif isinstance(data, list):list_data: List[Any] = data- _, list_string = purify_json_list(data=list_data, indent=self._log_config.json_logs_indent)+ _, list_string = purify_json_list(+ data=list_data,+ indent=self._log_config.json_logs_indent,+ is_truncate_bytes_enabled=True,+ )message = f"\n{list_string}"if title is not None:message = f"{title}:{message}"@@ -222,6 +226,7 @@ class LogDispatch:_, dict_string = purify_json(data=data,indent=self._log_config.json_logs_indent,+ is_truncate_bytes_enabled=True,is_warning_enabled=False,)message = f"\n{dict_string}"diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.pyindex 4df097c2b..6b4247bac 100644--- a/pipelex/tools/misc/json_utils.py+++ b/pipelex/tools/misc/json_utils.py@@ -6,6 +6,7 @@ from pydantic import BaseModelfrom pipelex.tools.exceptions import ToolExceptionfrom pipelex.tools.misc.file_utils import save_text_to_path+from pipelex.tools.typing.pydantic_utils import CustomBaseModelJsonContent = Union[Dict[Any, Any], List[Any]]@@ -194,7 +195,12 @@ def remove_none_values(json_content: JsonContent) -> JsonContent:return [remove_none_values(item) for item in json_content]-def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: bool = True) -> Tuple[Union[Dict[Any, Any], List[Any]], str]:+def purify_json(+ data: Any,+ indent: Optional[int] = None,+ is_truncate_bytes_enabled: bool = False,+ is_warning_enabled: bool = True,+) -> Tuple[Union[Dict[Any, Any], List[Any]], str]:"""Converts any Python object into a JSON-serializable format and its string representation.@@ -222,10 +228,18 @@ def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: boo'{"name": "test"}'"""dict_string: str- if isinstance(data, BaseModel):+ if isinstance(data, CustomBaseModel) and is_truncate_bytes_enabled:+ return purify_json(+ data.model_dump_truncated(serialize_as_any=True),+ indent=indent,+ is_truncate_bytes_enabled=is_truncate_bytes_enabled,+ is_warning_enabled=is_warning_enabled,+ )+ elif isinstance(data, BaseModel):return purify_json(data.model_dump(serialize_as_any=True),indent=indent,+ is_truncate_bytes_enabled=is_truncate_bytes_enabled,is_warning_enabled=is_warning_enabled,)@@ -233,7 +247,12 @@ def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: boothe_list = data # type: ignoreif not the_list:return [], "[]"- if isinstance(the_list[0], BaseModel):+ if isinstance(the_list[0], CustomBaseModel) and is_truncate_bytes_enabled:+ the_list_of_custom_base_models: List[CustomBaseModel] = the_list+ pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]+ dict_string = json.dumps(pure_list, indent=indent, default=str)+ return pure_list, dict_string+ elif isinstance(the_list[0], BaseModel):the_list_of_base_models: List[BaseModel] = the_listpure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]dict_string = json.dumps(pure_list, indent=indent, default=str)@@ -255,7 +274,11 @@ def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: booreturn pure_dict, dict_string-def purify_json_list(data: List[Any], indent: Optional[int] = None) -> Tuple[List[Any], str]:+def purify_json_list(+ data: List[Any],+ indent: Optional[int] = None,+ is_truncate_bytes_enabled: bool = False,+) -> Tuple[List[Any], str]:"""Converts a list of Python objects into a JSON-serializable list and its string representation.@@ -284,7 +307,12 @@ def purify_json_list(data: List[Any], indent: Optional[int] = None) -> Tuple[Lisif not data:return [], "[]"- if isinstance(data[0], BaseModel):+ if isinstance(data[0], CustomBaseModel) and is_truncate_bytes_enabled:+ the_list_of_custom_base_models: List[CustomBaseModel] = data+ pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]+ list_string = json.dumps(pure_list, indent=indent, default=str)+ return pure_list, list_string+ elif isinstance(data[0], BaseModel):the_list_of_base_models: List[BaseModel] = datapure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]list_string = json.dumps(pure_list, indent=indent, default=str)diff --git a/pipelex/tools/pdf/pypdfium2_renderer.py b/pipelex/tools/pdf/pypdfium2_renderer.pyindex 8917c7702..f6ec1bdf6 100644--- a/pipelex/tools/pdf/pypdfium2_renderer.py+++ b/pipelex/tools/pdf/pypdfium2_renderer.py@@ -12,6 +12,8 @@ from pipelex.tools.exceptions import ToolExceptionfrom pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx_asyncfrom pipelex.tools.misc.path_utils import clarify_path_or_url+PDFIUM2_REFERENCE_DPI = 72+class PyPdfium2RendererError(ToolException):pass@@ -36,7 +38,7 @@ class PyPdfium2Renderer:# ---- internal blocking helper ------------------------------------@staticmethod- def _render_pdf_pages_sync(pdf_input: PdfInput, dpi: float = 300) -> List[Image.Image]:+ def _render_pdf_pages_sync(pdf_input: PdfInput, scale: float) -> List[Image.Image]:pdf_doc = pdfium.PdfDocument(pdf_input)images: List[Image.Image] = []for index in range(len(pdf_doc)):@@ -44,7 +46,7 @@ class PyPdfium2Renderer:page = pdf_doc[index]pil_img: Image.Image = page.render( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]- scale=dpi / 72, # pyright: ignore[reportArgumentType]+ scale=scale, # pyright: ignore[reportArgumentType]force_bitmap_format=FPDFBitmap_BGRA, # always 4-channelrev_byteorder=True, # so we get RGBA).to_pil()@@ -55,12 +57,13 @@ class PyPdfium2Renderer:return images# ---- public async façade ------------------------------------------ async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int = 300) -> List[Image.Image]:+ async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int) -> List[Image.Image]:+ scale = dpi / PDFIUM2_REFERENCE_DPI"""Render *one* page and return PNG bytes."""async with self._pdfium_lock:- return await asyncio.to_thread(self._render_pdf_pages_sync, pdf_input, dpi)+ return await asyncio.to_thread(self._render_pdf_pages_sync, pdf_input, scale)- async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int = 300) -> List[Image.Image]:+ async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int) -> List[Image.Image]:pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignoreif pdf_url:pdf_bytes = await fetch_file_from_url_httpx_async(url=pdf_url)diff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.pyindex 499c3d4f1..f550ccfa3 100644--- a/pipelex/tools/typing/pydantic_utils.py+++ b/pipelex/tools/typing/pydantic_utils.py@@ -205,3 +205,57 @@ class CustomBaseModel(BaseModel):else:processed_args.append((name, value))return processed_args++ def model_dump_truncated(self, **kwargs: Any) -> Any:+ """+ Dump the model to a dictionary with serialize_as_any=True and apply+ AttributePolisher truncation to fields that should be truncated.+ Handles nested attributes recursively.++ Args:+ **kwargs: Additional keyword arguments to pass to model_dump++ Returns:+ Dictionary with truncated values where appropriate+ """+ # Get the model dump with serialize_as_any=True+ dumped_data = self.model_dump(**kwargs)++ # Apply truncation logic recursively+ return self._apply_truncation_recursive(dumped_data)++ def _apply_truncation_recursive(self, obj: Any, name: Optional[str] = None) -> Any:+ """+ Recursively apply AttributePolisher truncation logic to a data structure.++ Args:+ obj: The object to process+ name: The field name (for truncation logic)++ Returns:+ The processed object with truncation applied where appropriate+ """+ # First check if this specific object should be truncated+ if name and AttributePolisher.should_truncate(name=name, value=obj):+ return AttributePolisher.get_truncated_value(name, obj)++ # If it's a dictionary, recurse into its values+ if isinstance(obj, dict):+ obj_dict: Dict[str, Any] = obj+ truncated_dict: Dict[str, Any] = {}+ for key, value in obj_dict.items():+ truncated_dict[key] = self._apply_truncation_recursive(value, name=key)+ return truncated_dict++ # If it's a list, recurse into its items+ elif isinstance(obj, list):+ obj_list: List[Any] = obj+ return [self._apply_truncation_recursive(item, name=name) for item in obj_list]++ # If it's a tuple, recurse into its items and return as tuple+ elif isinstance(obj, tuple):+ return tuple(self._apply_truncation_recursive(item, name=name) for item in obj) # type: ignore++ # For all other types, return as-is+ else:+ return objdiff --git a/pyproject.toml b/pyproject.tomlindex f6cddd67f..62ba78892 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -201,7 +201,6 @@ addopts = "--import-mode=importlib -ra -m 'not (inference or llm or imgg or ocrasyncio_default_fixture_loop_scope = "session"markers = ["needs_output: tests that need output to be displayed",- "pipelex_api: tests that require access to the Pipelex API","inference: slow and costly due to inference calls","llm: slow and costly due to llm inference calls","imgg: slow and costly due to imgg inference calls",@@ -209,6 +208,7 @@ markers = ["gha_disabled: tests that should not run in GitHub Actions","codex_disabled: tests that should not run in Codex","dry_runnable: tests that can be run in dry-run mode",+ "pipelex_api: tests that require access to the Pipelex API",]minversion = "8.0"diff --git a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py b/tests/pipelex/pipelex_asynch/test_pipe_ocr.pyindex 349da9321..e30351675 100644--- a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py+++ b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py@@ -36,7 +36,7 @@ class TestPipeOCR:should_include_images=True,should_caption_images=False,should_include_page_views=True,- page_views_dpi=300,+ page_views_dpi=72,output_concept_code=NativeConcept.TEXT_AND_IMAGES.code,),pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),@@ -70,7 +70,7 @@ class TestPipeOCR:should_include_images=True,should_caption_images=False,should_include_page_views=True,- page_views_dpi=300,+ page_views_dpi=72,output_concept_code=NativeConcept.TEXT_AND_IMAGES.code,),pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),diff --git a/tests/tools/pdf/test_pdf.py b/tests/tools/pdf/test_pdf_render.pysimilarity index 61%rename from tests/tools/pdf/test_pdf.pyrename to tests/tools/pdf/test_pdf_render.pyindex 8e31425c4..d34175c81 100644--- a/tests/tools/pdf/test_pdf.py+++ b/tests/tools/pdf/test_pdf_render.py@@ -5,8 +5,8 @@ from tests.test_data import PDFTestCases@pytest.mark.asyncio(loop_scope="class")-class TestPDF:+class TestPdfRender:@pytest.mark.parametrize("file_path", PDFTestCases.DOCUMENT_FILE_PATHS)- async def test_pdf_path_to_images(self, file_path: str):- images = await pypdfium2_renderer.render_pdf_pages(file_path)+ async def test_render_pdf_from_path(self, file_path: str):+ images = await pypdfium2_renderer.render_pdf_pages(pdf_input=file_path, dpi=72)assert len(images) > 0