Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/pipelex get instance (#85)

Louis Choquelf0f9e5203fe0
diff --git a/README.md b/README.md
index 714da7918..32beb7973 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
<div align="center">
<a href="https://www.pipelex.com/"><img src="https://raw.githubusercontent.com/Pipelex/pipelex/main/.github/assets/logo.png" alt="Pipelex Logo" width="400" style="max-width: 100%; height: auto;"></a>
- <h2 align="center">Lean-code language for repeatable workflows</h2>
+ <h2 align="center">Open-source language for repeatable AI workflows</h2>
<p align="center">Pipelex is based on a simple declarative language that lets you define repeatable, structured, composable AI workflows.</p>
<div>
diff --git a/pipelex/cogt/config_cogt.py b/pipelex/cogt/config_cogt.py
index 8fb24e62a..aef295bb3 100644
--- a/pipelex/cogt/config_cogt.py
+++ b/pipelex/cogt/config_cogt.py
@@ -12,6 +12,7 @@ from pipelex.tools.config.models import ConfigModel
class OcrConfig(ConfigModel):
ocr_handles: List[str]
page_output_text_file_name: str
+ default_page_views_dpi: int
class ImggConfig(ConfigModel):
diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.py
index 89f3a2398..4e13e72a9 100644
--- a/pipelex/cogt/content_generation/content_generator.py
+++ b/pipelex/cogt/content_generation/content_generator.py
@@ -136,6 +136,7 @@ class ContentGenerator(ContentGeneratorProtocol):
object_class: Type[BaseModelTypeVar],
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
+ nb_items: Optional[int] = None,
wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
llm_assignment_for_object = LLMAssignment(
@@ -161,6 +162,7 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
+ nb_items: Optional[int] = None,
wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
llm_assignment_for_text = LLMAssignment(
diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.py
index e295359f3..538e8b6f5 100644
--- a/pipelex/cogt/content_generation/content_generator_dry.py
+++ b/pipelex/cogt/content_generation/content_generator_dry.py
@@ -63,6 +63,7 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
class ObjectFactory(ModelFactory[object_class]): # type: ignore
__model__ = object_class
+ __use_examples__ = True
obj = ObjectFactory.build()
return obj
@@ -96,24 +97,22 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
object_class: Type[BaseModelTypeVar],
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
+ nb_items: Optional[int] = None,
wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
func_name = "make_object_list_direct"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
- object_1 = await self.make_object_direct(
- job_metadata=job_metadata,
- object_class=object_class,
- llm_setting_for_object=llm_setting_for_object_list,
- llm_prompt_for_object=llm_prompt_for_object_list,
- )
- object_2 = await self.make_object_direct(
- job_metadata=job_metadata,
- object_class=object_class,
- llm_setting_for_object=llm_setting_for_object_list,
- llm_prompt_for_object=llm_prompt_for_object_list,
- )
- two_objects = [object_1, object_2]
- return two_objects
+ nb_list_items = nb_items or get_config().pipelex.dry_run_config.nb_list_items
+ objects = [
+ await self.make_object_direct(
+ job_metadata=job_metadata,
+ object_class=object_class,
+ llm_setting_for_object=llm_setting_for_object_list,
+ llm_prompt_for_object=llm_prompt_for_object_list,
+ )
+ for _ in range(nb_list_items)
+ ]
+ return objects
@override
@update_job_metadata
@@ -125,6 +124,7 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
+ nb_items: Optional[int] = None,
wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
func_name = "make_text_then_object_list"
@@ -134,6 +134,7 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
object_class=object_class,
llm_setting_for_object_list=llm_setting_for_object_list,
llm_prompt_for_object_list=llm_prompt_for_text,
+ nb_items=nb_items,
)
@override
@@ -149,8 +150,10 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
) -> GeneratedImage:
func_name = "make_single_image"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
+ image_urls = get_config().pipelex.dry_run_config.image_urls
+ image_url = image_urls[0]
generated_image = GeneratedImage(
- url="https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
+ url=image_url,
width=1536,
height=2752,
)
@@ -170,17 +173,14 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
) -> List[GeneratedImage]:
func_name = "make_image_list"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
+ image_urls = get_config().pipelex.dry_run_config.image_urls
generated_image_list = [
GeneratedImage(
- url="https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
+ url=image_urls[image_index % len(image_urls)],
width=1536,
height=2752,
- ),
- GeneratedImage(
- url="https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",
- width=1024,
- height=1536,
- ),
+ )
+ for image_index in range(nb_images)
]
return generated_image_list
@@ -225,25 +225,18 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
pages={1: ocr_image_as_page},
)
else:
- ocr_page_1 = Page(
- text="DRY RUN: OCR text",
- extracted_images=[],
- page_view=ExtractedImageFromPage(
- image_id="page_view_1",
- base_64="",
- caption="DRY RUN: OCR text",
- ),
- )
- ocr_page_2 = Page(
- text="DRY RUN: OCR text",
- extracted_images=[],
- page_view=ExtractedImageFromPage(
- image_id="page_view_2",
- base_64="",
- caption="DRY RUN: OCR text",
- ),
- )
- ocr_output = OcrOutput(
- pages={1: ocr_page_1, 2: ocr_page_2},
- )
+ nb_pages = get_config().pipelex.dry_run_config.nb_ocr_pages
+ pages = {
+ page_index: Page(
+ text="DRY RUN: OCR text",
+ extracted_images=[],
+ page_view=ExtractedImageFromPage(
+ image_id=f"page_view_{page_index}",
+ base_64="",
+ caption="DRY RUN: OCR text",
+ ),
+ )
+ for page_index in range(1, nb_pages + 1)
+ }
+ ocr_output = OcrOutput(pages=pages)
return ocr_output
diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.py
index 034badbd7..52ce35ec9 100644
--- a/pipelex/cogt/content_generation/content_generator_protocol.py
+++ b/pipelex/cogt/content_generation/content_generator_protocol.py
@@ -78,6 +78,7 @@ class ContentGeneratorProtocol(Protocol):
object_class: Type[BaseModelTypeVar],
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
+ nb_items: Optional[int] = None,
wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]: ...
@@ -89,6 +90,7 @@ class ContentGeneratorProtocol(Protocol):
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
+ nb_items: Optional[int] = None,
wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]: ...
diff --git a/pipelex/cogt/ocr/ocr_job_components.py b/pipelex/cogt/ocr/ocr_job_components.py
index 7a06cfdd3..f74f322e8 100644
--- a/pipelex/cogt/ocr/ocr_job_components.py
+++ b/pipelex/cogt/ocr/ocr_job_components.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
from pydantic import BaseModel
from pipelex.tools.config.models import ConfigModel
@@ -7,7 +9,7 @@ class OcrJobParams(BaseModel):
should_include_images: bool
should_caption_images: bool
should_include_page_views: bool
- page_views_dpi: int
+ page_views_dpi: Optional[int] = None
@classmethod
def make_default_ocr_job_params(cls) -> "OcrJobParams":
@@ -15,7 +17,7 @@ class OcrJobParams(BaseModel):
should_caption_images=False,
should_include_page_views=False,
should_include_images=True,
- page_views_dpi=300,
+ page_views_dpi=None,
)
diff --git a/pipelex/config.py b/pipelex/config.py
index b2373df4a..87005e265 100644
--- a/pipelex/config.py
+++ b/pipelex/config.py
@@ -5,7 +5,7 @@ from pydantic import Field, field_validator
from pipelex.cogt.config_cogt import Cogt
from pipelex.cogt.llm.llm_models.llm_prompting_target import LLMPromptingTarget
-from pipelex.exceptions import PipelexError, StaticValidationErrorType
+from pipelex.exceptions import PipelexConfigError, StaticValidationErrorType
from pipelex.hub import get_required_config
from pipelex.libraries.library_config import LibraryConfig
from pipelex.pipeline.track.tracker_config import TrackerConfig
@@ -40,10 +40,6 @@ class StaticValidationConfig(ConfigModel):
return the_dict
-class PipelexConfigError(PipelexError):
- pass
-
-
class PipeRunConfig(ConfigModel):
pipe_stack_limit: int
@@ -51,6 +47,15 @@ class PipeRunConfig(ConfigModel):
class DryRunConfig(ConfigModel):
apply_to_jinja2_rendering: bool
text_gen_truncate_length: int
+ nb_list_items: int
+ nb_ocr_pages: int
+ image_urls: List[str]
+
+ @field_validator("image_urls", mode="before")
+ def validate_image_urls(cls, value: List[str]) -> List[str]:
+ if not value:
+ raise PipelexConfigError("dry_run_config.image_urls must be a non-empty list")
+ return value
class GenericTemplateNames(ConfigModel):
diff --git a/pipelex/core/domain_library.py b/pipelex/core/domain_library.py
index 8225ec972..f2793042b 100644
--- a/pipelex/core/domain_library.py
+++ b/pipelex/core/domain_library.py
@@ -3,7 +3,7 @@ from typing import Dict, List, Optional
from pydantic import Field, RootModel
from typing_extensions import override
-from pipelex import log
+from pipelex import log, pretty_print
from pipelex.core.domain import Domain
from pipelex.core.domain_provider_abstract import DomainProviderAbstract
from pipelex.exceptions import DomainLibraryError
@@ -21,11 +21,13 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
def reset(self):
self.root = {}
- def add_new_domain(self, domain: Domain):
+ def add_domain_details(self, domain: Domain):
domain_code = domain.code
- if domain_code in self.root:
- raise DomainLibraryError(f"Domain '{domain_code}' already exists in the library")
- self.root[domain_code] = domain
+ if existing_domain := self.root.get(domain_code):
+ # merge the new domain with the existing one
+ self.root[domain_code] = existing_domain.model_copy(update=domain.model_dump())
+ else:
+ self.root[domain_code] = domain
def print_all(self):
log.dev("-" * 80)
@@ -58,6 +60,7 @@ class DomainLibrary(RootModel[DomainLibraryRoot], DomainProviderAbstract):
def get_required_domain(self, domain_code: str) -> Domain:
the_domain = self.get_domain(domain_code=domain_code)
if not the_domain:
+ pretty_print(self.root, title="DomainLibrary")
raise DomainLibraryError(f"Domain '{domain_code}' not found. Check for typos and make sure it is declared in a pipeline library.")
return the_domain
diff --git a/pipelex/core/pipe_run_params.py b/pipelex/core/pipe_run_params.py
index 20df46d41..1049503e6 100644
--- a/pipelex/core/pipe_run_params.py
+++ b/pipelex/core/pipe_run_params.py
@@ -18,6 +18,8 @@ class PipeRunMode(StrEnum):
DRY = "dry"
+FORCE_DRY_RUN_MODE_ENV_KEY = "PIPELEX_FORCE_DRY_RUN_MODE"
+
PipeOutputMultiplicity = Union[bool, int]
diff --git a/pipelex/core/stuff.py b/pipelex/core/stuff.py
index 4d4e211d8..48e0fbeb9 100644
--- a/pipelex/core/stuff.py
+++ b/pipelex/core/stuff.py
@@ -20,13 +20,14 @@ from pipelex.core.stuff_content import (
)
from pipelex.exceptions import StuffError
from pipelex.tools.misc.string_utils import pascal_case_to_snake_case
+from pipelex.tools.typing.pydantic_utils import CustomBaseModel
class StuffCreationRecord(BaseModel):
pass
-class Stuff(BaseModel):
+class Stuff(CustomBaseModel):
model_config = ConfigDict(extra="ignore", strict=True)
stuff_code: str
diff --git a/pipelex/libraries/library_manager.py b/pipelex/libraries/library_manager.py
index f527f9f75..eaa7fce2d 100644
--- a/pipelex/libraries/library_manager.py
+++ b/pipelex/libraries/library_manager.py
@@ -133,9 +133,6 @@ class LibraryManager:
"Just write 'domain = \"my_domain\"' at the top of the file."
)
domain_definition = library_dict.get("definition")
- if domain_definition is None:
- # we skip the domain without definition, it must be defined one and only one time in the domain library
- continue
system_prompt = library_dict.get("system_prompt")
system_prompt_to_structure = library_dict.get("system_prompt_to_structure")
prompt_template_to_structure = library_dict.get("prompt_template_to_structure")
@@ -146,7 +143,7 @@ class LibraryManager:
system_prompt_to_structure=system_prompt_to_structure,
prompt_template_to_structure=prompt_template_to_structure,
)
- self.domain_library.add_new_domain(domain=domain)
+ self.domain_library.add_domain_details(domain=domain)
# Second pass: load all concepts
for toml_path in toml_file_paths:
diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.py
index 1a25c16da..62b510e67 100644
--- a/pipelex/pipe_operators/pipe_llm.py
+++ b/pipelex/pipe_operators/pipe_llm.py
@@ -412,6 +412,7 @@ class PipeLLM(PipeOperator):
llm_setting_main=self.llm_setting_main,
llm_prompt_factory_for_object_list=llm_prompt_2_factory,
llm_setting_for_object_list=self.llm_setting_for_object_list,
+ nb_items=fixed_nb_output,
wfid=task_desc,
)
else:
@@ -423,6 +424,7 @@ class PipeLLM(PipeOperator):
object_class=content_class,
llm_prompt_for_object_list=llm_prompt_1,
llm_setting_for_object_list=self.llm_setting_for_object_list_direct,
+ nb_items=fixed_nb_output,
wfid=task_desc,
)
diff --git a/pipelex/pipe_operators/pipe_ocr_factory.py b/pipelex/pipe_operators/pipe_ocr_factory.py
index 6d4e7290c..19ba7a48b 100644
--- a/pipelex/pipe_operators/pipe_ocr_factory.py
+++ b/pipelex/pipe_operators/pipe_ocr_factory.py
@@ -4,6 +4,7 @@ from typing_extensions import override
from pipelex.cogt.ocr.ocr_engine_factory import OcrEngineFactory, OcrPlatform
from pipelex.cogt.ocr.ocr_handle import OcrHandle
+from pipelex.config import get_config
from pipelex.core.pipe_blueprint import PipeBlueprint, PipeSpecificFactoryProtocol
from pipelex.core.pipe_input_spec import PipeInputSpec
from pipelex.pipe_operators.pipe_ocr import PipeOcr
@@ -15,7 +16,7 @@ class PipeOcrBlueprint(PipeBlueprint):
page_images: bool = False
page_image_captions: bool = False
page_views: bool = False
- page_views_dpi: int = 300
+ page_views_dpi: Optional[int] = None
class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
@@ -42,7 +43,7 @@ class PipeOcrFactory(PipeSpecificFactoryProtocol[PipeOcrBlueprint, PipeOcr]):
should_include_images=pipe_blueprint.page_images,
should_caption_images=pipe_blueprint.page_image_captions,
should_include_page_views=pipe_blueprint.page_views,
- page_views_dpi=pipe_blueprint.page_views_dpi,
+ page_views_dpi=pipe_blueprint.page_views_dpi or get_config().cogt.ocr_config.default_page_views_dpi,
)
@classmethod
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 50a3d5a82..a0c75361a 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -243,3 +243,9 @@ class Pipelex:
pipelex_instance.finish_setup()
log.info(f"Pipelex {PACKAGE_VERSION} initialized.")
return pipelex_instance
+
+ @classmethod
+ def get_instance(cls) -> Self:
+ if cls._pipelex_instance is None:
+ raise RuntimeError("Pipelex is not initialized")
+ return cls._pipelex_instance
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 79431a752..345f6f142 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -128,6 +128,7 @@ seed = "auto"
[cogt.ocr_config]
ocr_handles = ["mistral/mistral-ocr-latest"]
page_output_text_file_name = "page_text.md"
+default_page_views_dpi = 72
####################################################################################################
# Plugins config
@@ -245,3 +246,10 @@ pipe_stack_limit = 20
[pipelex.dry_run_config]
apply_to_jinja2_rendering = false
text_gen_truncate_length = 256
+nb_list_items = 3
+nb_ocr_pages = 4
+image_urls = [
+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_1.jpg",
+ "https://storage.googleapis.com/public_test_files_7fa6_4277_9ab/fashion/fashion_photo_2.png",
+ ]
+
diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.py
index 06db5bded..b83b9dfd0 100644
--- a/pipelex/pipeline/execute.py
+++ b/pipelex/pipeline/execute.py
@@ -2,12 +2,13 @@ from typing import Optional, Tuple
from pipelex import pretty_print
from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunMode
+from pipelex.core.pipe_run_params import FORCE_DRY_RUN_MODE_ENV_KEY, PipeOutputMultiplicity, PipeRunMode
from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.working_memory import WorkingMemory
from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from pipelex.pipeline.job_metadata import JobMetadata
+from pipelex.tools.environment import get_optional_env
async def execute_pipeline(
@@ -16,7 +17,7 @@ async def execute_pipeline(
output_name: Optional[str] = None,
output_multiplicity: Optional[PipeOutputMultiplicity] = None,
dynamic_output_concept_code: Optional[str] = None,
- pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,
+ pipe_run_mode: Optional[PipeRunMode] = None,
) -> Tuple[PipeOutput, str]:
"""Execute a pipeline and wait for its completion.
@@ -37,13 +38,22 @@ async def execute_pipeline(
dynamic_output_concept_code:
Override the dynamic output concept code.
pipe_run_mode:
- Pipe run mode: ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``.
+ Pipe run mode: if specified, it must be ``PipeRunMode.LIVE`` or ``PipeRunMode.DRY``.
+ If not specified, the pipe run mode is inferred from the environment variable
+ ``PIPELEX_FORCE_DRY_RUN_MODE``. If the environment variable is not set,
+ the pipe run mode is ``PipeRunMode.LIVE``.
Returns
-------
Tuple[PipeOutput, str]
A tuple containing the pipe output and the pipeline run ID.
"""
+ if pipe_run_mode is None:
+ if run_mode_from_env := get_optional_env(key=FORCE_DRY_RUN_MODE_ENV_KEY):
+ pipe_run_mode = PipeRunMode(run_mode_from_env)
+ else:
+ pipe_run_mode = PipeRunMode.LIVE
+
pipeline = get_pipeline_manager().add_new_pipeline()
pipeline_run_id = pipeline.pipeline_run_id
get_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)
diff --git a/pipelex/tools/log/log_dispatch.py b/pipelex/tools/log/log_dispatch.py
index db50ec652..f5031925e 100644
--- a/pipelex/tools/log/log_dispatch.py
+++ b/pipelex/tools/log/log_dispatch.py
@@ -209,7 +209,11 @@ class LogDispatch:
self._log_to_console(message=message, severity=severity)
elif isinstance(data, list):
list_data: List[Any] = data
- _, list_string = purify_json_list(data=list_data, indent=self._log_config.json_logs_indent)
+ _, list_string = purify_json_list(
+ data=list_data,
+ indent=self._log_config.json_logs_indent,
+ is_truncate_bytes_enabled=True,
+ )
message = f"\n{list_string}"
if title is not None:
message = f"{title}:{message}"
@@ -222,6 +226,7 @@ class LogDispatch:
_, dict_string = purify_json(
data=data,
indent=self._log_config.json_logs_indent,
+ is_truncate_bytes_enabled=True,
is_warning_enabled=False,
)
message = f"\n{dict_string}"
diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.py
index 4df097c2b..6b4247bac 100644
--- a/pipelex/tools/misc/json_utils.py
+++ b/pipelex/tools/misc/json_utils.py
@@ -6,6 +6,7 @@ from pydantic import BaseModel
from pipelex.tools.exceptions import ToolException
from pipelex.tools.misc.file_utils import save_text_to_path
+from pipelex.tools.typing.pydantic_utils import CustomBaseModel
JsonContent = Union[Dict[Any, Any], List[Any]]
@@ -194,7 +195,12 @@ def remove_none_values(json_content: JsonContent) -> JsonContent:
return [remove_none_values(item) for item in json_content]
-def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: bool = True) -> Tuple[Union[Dict[Any, Any], List[Any]], str]:
+def purify_json(
+ data: Any,
+ indent: Optional[int] = None,
+ is_truncate_bytes_enabled: bool = False,
+ is_warning_enabled: bool = True,
+) -> Tuple[Union[Dict[Any, Any], List[Any]], str]:
"""
Converts any Python object into a JSON-serializable format and its string representation.
@@ -222,10 +228,18 @@ def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: boo
'{"name": "test"}'
"""
dict_string: str
- if isinstance(data, BaseModel):
+ if isinstance(data, CustomBaseModel) and is_truncate_bytes_enabled:
+ return purify_json(
+ data.model_dump_truncated(serialize_as_any=True),
+ indent=indent,
+ is_truncate_bytes_enabled=is_truncate_bytes_enabled,
+ is_warning_enabled=is_warning_enabled,
+ )
+ elif isinstance(data, BaseModel):
return purify_json(
data.model_dump(serialize_as_any=True),
indent=indent,
+ is_truncate_bytes_enabled=is_truncate_bytes_enabled,
is_warning_enabled=is_warning_enabled,
)
@@ -233,7 +247,12 @@ def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: boo
the_list = data # type: ignore
if not the_list:
return [], "[]"
- if isinstance(the_list[0], BaseModel):
+ if isinstance(the_list[0], CustomBaseModel) and is_truncate_bytes_enabled:
+ the_list_of_custom_base_models: List[CustomBaseModel] = the_list
+ pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]
+ dict_string = json.dumps(pure_list, indent=indent, default=str)
+ return pure_list, dict_string
+ elif isinstance(the_list[0], BaseModel):
the_list_of_base_models: List[BaseModel] = the_list
pure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]
dict_string = json.dumps(pure_list, indent=indent, default=str)
@@ -255,7 +274,11 @@ def purify_json(data: Any, indent: Optional[int] = None, is_warning_enabled: boo
return pure_dict, dict_string
-def purify_json_list(data: List[Any], indent: Optional[int] = None) -> Tuple[List[Any], str]:
+def purify_json_list(
+ data: List[Any],
+ indent: Optional[int] = None,
+ is_truncate_bytes_enabled: bool = False,
+) -> Tuple[List[Any], str]:
"""
Converts a list of Python objects into a JSON-serializable list and its string representation.
@@ -284,7 +307,12 @@ def purify_json_list(data: List[Any], indent: Optional[int] = None) -> Tuple[Lis
if not data:
return [], "[]"
- if isinstance(data[0], BaseModel):
+ if isinstance(data[0], CustomBaseModel) and is_truncate_bytes_enabled:
+ the_list_of_custom_base_models: List[CustomBaseModel] = data
+ pure_list = [item.model_dump_truncated(serialize_as_any=True) for item in the_list_of_custom_base_models]
+ list_string = json.dumps(pure_list, indent=indent, default=str)
+ return pure_list, list_string
+ elif isinstance(data[0], BaseModel):
the_list_of_base_models: List[BaseModel] = data
pure_list = [item.model_dump(serialize_as_any=True) for item in the_list_of_base_models]
list_string = json.dumps(pure_list, indent=indent, default=str)
diff --git a/pipelex/tools/pdf/pypdfium2_renderer.py b/pipelex/tools/pdf/pypdfium2_renderer.py
index 8917c7702..f6ec1bdf6 100644
--- a/pipelex/tools/pdf/pypdfium2_renderer.py
+++ b/pipelex/tools/pdf/pypdfium2_renderer.py
@@ -12,6 +12,8 @@ from pipelex.tools.exceptions import ToolException
from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx_async
from pipelex.tools.misc.path_utils import clarify_path_or_url
+PDFIUM2_REFERENCE_DPI = 72
+
class PyPdfium2RendererError(ToolException):
pass
@@ -36,7 +38,7 @@ class PyPdfium2Renderer:
# ---- internal blocking helper ------------------------------------
@staticmethod
- def _render_pdf_pages_sync(pdf_input: PdfInput, dpi: float = 300) -> List[Image.Image]:
+ def _render_pdf_pages_sync(pdf_input: PdfInput, scale: float) -> List[Image.Image]:
pdf_doc = pdfium.PdfDocument(pdf_input)
images: List[Image.Image] = []
for index in range(len(pdf_doc)):
@@ -44,7 +46,7 @@ class PyPdfium2Renderer:
page = pdf_doc[index]
pil_img: Image.Image = page.render( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
- scale=dpi / 72, # pyright: ignore[reportArgumentType]
+ scale=scale, # pyright: ignore[reportArgumentType]
force_bitmap_format=FPDFBitmap_BGRA, # always 4-channel
rev_byteorder=True, # so we get RGBA
).to_pil()
@@ -55,12 +57,13 @@ class PyPdfium2Renderer:
return images
# ---- public async façade -----------------------------------------
- async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int = 300) -> List[Image.Image]:
+ async def render_pdf_pages(self, pdf_input: PdfInput, dpi: int) -> List[Image.Image]:
+ scale = dpi / PDFIUM2_REFERENCE_DPI
"""Render *one* page and return PNG bytes."""
async with self._pdfium_lock:
- return await asyncio.to_thread(self._render_pdf_pages_sync, pdf_input, dpi)
+ return await asyncio.to_thread(self._render_pdf_pages_sync, pdf_input, scale)
- async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int = 300) -> List[Image.Image]:
+ async def render_pdf_pages_from_uri(self, pdf_uri: str, dpi: int) -> List[Image.Image]:
pdf_path, pdf_url = clarify_path_or_url(path_or_uri=pdf_uri) # pyright: ignore
if pdf_url:
pdf_bytes = await fetch_file_from_url_httpx_async(url=pdf_url)
diff --git a/pipelex/tools/typing/pydantic_utils.py b/pipelex/tools/typing/pydantic_utils.py
index 499c3d4f1..f550ccfa3 100644
--- a/pipelex/tools/typing/pydantic_utils.py
+++ b/pipelex/tools/typing/pydantic_utils.py
@@ -205,3 +205,57 @@ class CustomBaseModel(BaseModel):
else:
processed_args.append((name, value))
return processed_args
+
+ def model_dump_truncated(self, **kwargs: Any) -> Any:
+ """
+ Dump the model to a dictionary with serialize_as_any=True and apply
+ AttributePolisher truncation to fields that should be truncated.
+ Handles nested attributes recursively.
+
+ Args:
+ **kwargs: Additional keyword arguments to pass to model_dump
+
+ Returns:
+ Dictionary with truncated values where appropriate
+ """
+ # Get the model dump with serialize_as_any=True
+ dumped_data = self.model_dump(**kwargs)
+
+ # Apply truncation logic recursively
+ return self._apply_truncation_recursive(dumped_data)
+
+ def _apply_truncation_recursive(self, obj: Any, name: Optional[str] = None) -> Any:
+ """
+ Recursively apply AttributePolisher truncation logic to a data structure.
+
+ Args:
+ obj: The object to process
+ name: The field name (for truncation logic)
+
+ Returns:
+ The processed object with truncation applied where appropriate
+ """
+ # First check if this specific object should be truncated
+ if name and AttributePolisher.should_truncate(name=name, value=obj):
+ return AttributePolisher.get_truncated_value(name, obj)
+
+ # If it's a dictionary, recurse into its values
+ if isinstance(obj, dict):
+ obj_dict: Dict[str, Any] = obj
+ truncated_dict: Dict[str, Any] = {}
+ for key, value in obj_dict.items():
+ truncated_dict[key] = self._apply_truncation_recursive(value, name=key)
+ return truncated_dict
+
+ # If it's a list, recurse into its items
+ elif isinstance(obj, list):
+ obj_list: List[Any] = obj
+ return [self._apply_truncation_recursive(item, name=name) for item in obj_list]
+
+ # If it's a tuple, recurse into its items and return as tuple
+ elif isinstance(obj, tuple):
+ return tuple(self._apply_truncation_recursive(item, name=name) for item in obj) # type: ignore
+
+ # For all other types, return as-is
+ else:
+ return obj
diff --git a/pyproject.toml b/pyproject.toml
index f6cddd67f..62ba78892 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -201,7 +201,6 @@ addopts = "--import-mode=importlib -ra -m 'not (inference or llm or imgg or ocr
asyncio_default_fixture_loop_scope = "session"
markers = [
"needs_output: tests that need output to be displayed",
- "pipelex_api: tests that require access to the Pipelex API",
"inference: slow and costly due to inference calls",
"llm: slow and costly due to llm inference calls",
"imgg: slow and costly due to imgg inference calls",
@@ -209,6 +208,7 @@ markers = [
"gha_disabled: tests that should not run in GitHub Actions",
"codex_disabled: tests that should not run in Codex",
"dry_runnable: tests that can be run in dry-run mode",
+ "pipelex_api: tests that require access to the Pipelex API",
]
minversion = "8.0"
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py
index 349da9321..e30351675 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py
@@ -36,7 +36,7 @@ class TestPipeOCR:
should_include_images=True,
should_caption_images=False,
should_include_page_views=True,
- page_views_dpi=300,
+ page_views_dpi=72,
output_concept_code=NativeConcept.TEXT_AND_IMAGES.code,
),
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
@@ -70,7 +70,7 @@ class TestPipeOCR:
should_include_images=True,
should_caption_images=False,
should_include_page_views=True,
- page_views_dpi=300,
+ page_views_dpi=72,
output_concept_code=NativeConcept.TEXT_AND_IMAGES.code,
),
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
diff --git a/tests/tools/pdf/test_pdf.py b/tests/tools/pdf/test_pdf_render.py
similarity index 61%
rename from tests/tools/pdf/test_pdf.py
rename to tests/tools/pdf/test_pdf_render.py
index 8e31425c4..d34175c81 100644
--- a/tests/tools/pdf/test_pdf.py
+++ b/tests/tools/pdf/test_pdf_render.py
@@ -5,8 +5,8 @@ from tests.test_data import PDFTestCases
@pytest.mark.asyncio(loop_scope="class")
-class TestPDF:
+class TestPdfRender:
@pytest.mark.parametrize("file_path", PDFTestCases.DOCUMENT_FILE_PATHS)
- async def test_pdf_path_to_images(self, file_path: str):
- images = await pypdfium2_renderer.render_pdf_pages(file_path)
+ async def test_render_pdf_from_path(self, file_path: str):
+ images = await pypdfium2_renderer.render_pdf_pages(pdf_input=file_path, dpi=72)
assert len(images) > 0