Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Refactor/cleanup setup and dependency injections (#381)
Louis Choquelade692d48439
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.mdindex 5a54470dd..b467128ac 100644--- a/.github/copilot-instructions.md+++ b/.github/copilot-instructions.md@@ -1,6 +1,4 @@<!-- BEGIN_PIPELEX_RULES -->-# Pipelex Rules-## Guide to write or edit pipelines using the Pipelex language in .plx files- Always first write your "plan" in natural language, then transcribe it in pipelex.diff --git a/.windsurfrules.md b/.windsurfrules.mdindex 5a54470dd..b467128ac 100644--- a/.windsurfrules.md+++ b/.windsurfrules.md@@ -1,6 +1,4 @@<!-- BEGIN_PIPELEX_RULES -->-# Pipelex Rules-## Guide to write or edit pipelines using the Pipelex language in .plx files- Always first write your "plan" in natural language, then transcribe it in pipelex.diff --git a/AGENTS.md b/AGENTS.mdindex 5a54470dd..b467128ac 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -1,6 +1,4 @@<!-- BEGIN_PIPELEX_RULES -->-# Pipelex Rules-## Guide to write or edit pipelines using the Pipelex language in .plx files- Always first write your "plan" in natural language, then transcribe it in pipelex.diff --git a/BLACKBOX_RULES.md b/BLACKBOX_RULES.mdindex 5a54470dd..b467128ac 100644--- a/BLACKBOX_RULES.md+++ b/BLACKBOX_RULES.md@@ -1,6 +1,4 @@<!-- BEGIN_PIPELEX_RULES -->-# Pipelex Rules-## Guide to write or edit pipelines using the Pipelex language in .plx files- Always first write your "plan" in natural language, then transcribe it in pipelex.diff --git a/CLAUDE.md b/CLAUDE.mdindex 5a54470dd..b467128ac 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -1,6 +1,4 @@<!-- BEGIN_PIPELEX_RULES -->-# Pipelex Rules-## Guide to write or edit pipelines using the Pipelex language in .plx files- Always first write your "plan" in natural language, then transcribe it in pipelex.diff --git a/docs/pages/advanced-customization/observer-provider-injection.md b/docs/pages/advanced-customization/observer-provider-injection.mdindex ab71f0d13..3f45bfb5f 100644--- a/docs/pages/advanced-customization/observer-provider-injection.md+++ b/docs/pages/advanced-customization/observer-provider-injection.md@@ -107,7 +107,7 @@ my_observer = MyCustomObserver(config_param="production")# Inject it into the hubhub = get_pipelex_hub()-hub.set_observer_provider(my_observer)+hub.set_observer(my_observer)```### Step 3: Ensure Proper Integration@@ -132,7 +132,7 @@ local_observer = LocalObserver()local_observer = LocalObserver(storage_dir="/path/to/custom/dir")# Inject the observer-get_pipelex_hub().set_observer_provider(local_observer)+get_pipelex_hub().set_observer(local_observer)```The LocalObserver creates separate JSONL files for each event type:@@ -209,7 +209,7 @@ def setup_pipelex():# Setup custom observerobserver = MyCustomObserver()- get_pipelex_hub().set_observer_provider(observer)+ get_pipelex_hub().set_observer(observer)return pipelex_instance```diff --git a/pipelex/builder/pipe/pipe_llm_spec.py b/pipelex/builder/pipe/pipe_llm_spec.pyindex d0d30ffba..03bb5d864 100644--- a/pipelex/builder/pipe/pipe_llm_spec.py+++ b/pipelex/builder/pipe/pipe_llm_spec.py@@ -21,7 +21,6 @@ class LLMSkill(StrEnum):LLM_FOR_VISUAL_ANALYSIS = "llm_for_visual_analysis"LLM_FOR_VISUAL_DESIGN = "llm_for_visual_design"LLM_FOR_CREATIVE_WRITING = "llm_for_creative_writing"- LLM_FOR_COMPLEX_REASONING = "llm_for_complex_reasoning"LLM_TO_REASON_ON_DIAGRAM = "llm_to_reason_on_diagram"LLM_TO_ANALYZE_DATA = "llm_to_analyze_data"LLM_TO_CODE = "llm_to_code"diff --git a/pipelex/hub.py b/pipelex/hub.pyindex b5f7d024d..d8a851d33 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -22,7 +22,6 @@ from pipelex.core.pipes.pipe_library_abstract import PipeLibraryAbstractfrom pipelex.libraries.library_manager_abstract import LibraryManagerAbstractfrom pipelex.observer.observer_protocol import ObserverProtocolfrom pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocolfrom pipelex.pipeline.pipeline import Pipelinefrom pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstractfrom pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocol@@ -65,8 +64,7 @@ class PipelexHub:# pipelineself._pipeline_tracker: PipelineTrackerProtocol | None = Noneself._pipeline_manager: PipelineManagerAbstract | None = None- self._activity_manager: ActivityManagerProtocol | None = None- self._observer_provider: ObserverProtocol | None = None+ self._observer: ObserverProtocol | None = None############################################################# Class methods for singleton management@@ -155,14 +153,11 @@ class PipelexHub:def set_pipeline_manager(self, pipeline_manager: PipelineManagerAbstract):self._pipeline_manager = pipeline_manager- def set_activity_manager(self, activity_manager: ActivityManagerProtocol):- self._activity_manager = activity_manager-def set_library_manager(self, library_manager: LibraryManagerAbstract):self._library_manager = library_manager- def set_observer_provider(self, observer_provider: ObserverProtocol):- self._observer_provider = observer_provider+ def set_observer(self, observer: ObserverProtocol):+ self._observer = observer############################################################# Getters@@ -269,23 +264,17 @@ class PipelexHub:raise RuntimeError(msg)return self._pipeline_manager- def get_activity_manager(self) -> ActivityManagerProtocol:- if self._activity_manager is None:- msg = "Activity manager is not set. You must initialize Pipelex first."- raise RuntimeError(msg)- return self._activity_manager-def get_required_library_manager(self) -> LibraryManagerAbstract:if self._library_manager is None:msg = "Library manager is not set. You must initialize Pipelex first."raise RuntimeError(msg)return self._library_manager- def get_observer_provider(self) -> ObserverProtocol:- if self._observer_provider is None:+ def get_observer(self) -> ObserverProtocol:+ if self._observer is None:msg = "Observer is not set. You must initialize Pipelex first."raise RuntimeError(msg)- return self._observer_provider+ return self._observer# Shorthand functions for accessing the singleton@@ -416,10 +405,6 @@ def get_pipeline_manager() -> PipelineManagerAbstract:return get_pipelex_hub().get_required_pipeline_manager()-def get_activity_manager() -> ActivityManagerProtocol:- return get_pipelex_hub().get_activity_manager()--def get_pipeline(pipeline_run_id: str) -> Pipeline:return get_pipeline_manager().get_pipeline(pipeline_run_id=pipeline_run_id)@@ -428,8 +413,8 @@ def get_library_manager() -> LibraryManagerAbstract:return get_pipelex_hub().get_required_library_manager()-def get_observer_provider() -> ObserverProtocol:- return get_pipelex_hub().get_observer_provider()+def get_observer() -> ObserverProtocol:+ return get_pipelex_hub().get_observer()def get_native_concept(native_concept: NativeConceptCode) -> Concept:diff --git a/pipelex/kit/index.toml b/pipelex/kit/index.tomlindex 148f5aa4f..eff4114d4 100644--- a/pipelex/kit/index.toml+++ b/pipelex/kit/index.toml@@ -63,35 +63,30 @@ globs = ["*.plx", "*.toml"][agent_rules.targets.agents]path = "AGENTS.md"-strategy = "merge"marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"marker_end = "<!-- END_PIPELEX_RULES -->"heading_1 = "# Pipelex Rules"[agent_rules.targets.claude]path = "CLAUDE.md"-strategy = "merge"marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"marker_end = "<!-- END_PIPELEX_RULES -->"heading_1 = "# Pipelex Rules"[agent_rules.targets.github_copilot]path = ".github/copilot-instructions.md"-strategy = "merge"marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"marker_end = "<!-- END_PIPELEX_RULES -->"heading_1 = "# Pipelex Rules"[agent_rules.targets.windsurf]path = ".windsurfrules.md"-strategy = "merge"marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"marker_end = "<!-- END_PIPELEX_RULES -->"heading_1 = "# Pipelex Rules"[agent_rules.targets.blackbox]path = "BLACKBOX_RULES.md"-strategy = "merge"marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"marker_end = "<!-- END_PIPELEX_RULES -->"heading_1 = "# Pipelex Rules"diff --git a/pipelex/kit/index_models.py b/pipelex/kit/index_models.pyindex a74881181..9e910ac4a 100644--- a/pipelex/kit/index_models.py+++ b/pipelex/kit/index_models.py@@ -22,7 +22,6 @@ class Target(ConfigModel):"""Configuration for a single-file merge target."""path: str = Field(description="Path to the target file relative to repo root")- strategy: str = Field(description="Merge strategy (currently only 'merge' supported)")marker_begin: str = Field(description="Beginning marker for content insertion")marker_end: str = Field(description="Ending marker for content insertion")heading_1: str | None = Field(default=None, description="Main title (H1) to add when inserting into empty file or file with no H1 headings")diff --git a/pipelex/observer/multi_observer.py b/pipelex/observer/multi_observer.pynew file mode 100644index 000000000..206bb1958--- /dev/null+++ b/pipelex/observer/multi_observer.py@@ -0,0 +1,31 @@+from typing_extensions import override++from pipelex.observer.observer_protocol import ObserverProtocol, PayloadType+++class MultiObserver(ObserverProtocol):+ def __init__(self, observers: dict[str, ObserverProtocol] | None = None) -> None:+ self.observers: dict[str, ObserverProtocol] = observers or {}++ def add_observer(self, name: str, observer: ObserverProtocol) -> None:+ """Add an observer with a given name."""+ self.observers[name] = observer++ def remove_observer(self, name: str) -> None:+ """Remove an observer by name."""+ self.observers.pop(name, None)++ @override+ async def observe_before_run(self, payload: PayloadType) -> None:+ for observer in self.observers.values():+ await observer.observe_before_run(payload)++ @override+ async def observe_after_successful_run(self, payload: PayloadType) -> None:+ for observer in self.observers.values():+ await observer.observe_after_successful_run(payload)++ @override+ async def observe_after_failing_run(self, payload: PayloadType) -> None:+ for observer in self.observers.values():+ await observer.observe_after_failing_run(payload)diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.pyindex 8db54169b..a49103871 100644--- a/pipelex/pipe_operators/pipe_operator.py+++ b/pipelex/pipe_operators/pipe_operator.py@@ -8,9 +8,7 @@ from pipelex.core.memory.working_memory import WorkingMemoryfrom pipelex.core.pipes.pipe_abstract import PipeAbstractfrom pipelex.core.pipes.pipe_output import PipeOutputfrom pipelex.core.stuffs.text_content import TextContent-from pipelex.hub import get_activity_managerfrom pipelex.pipe_run.pipe_run_params import PipeRunMode, PipeRunParams-from pipelex.pipeline.activity.activity_models import ActivityReportfrom pipelex.pipeline.job_metadata import JobMetadataPipeOperatorOutputType = TypeVar("PipeOperatorOutputType", bound=PipeOutput)@@ -70,12 +68,6 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):pipe_run_params=pipe_run_params,output_name=output_name,)- get_activity_manager().dispatch_activity(- activity_report=ActivityReport(- job_metadata=job_metadata,- content=pipe_output.main_stuff,- ),- )pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)diff --git a/pipelex/pipe_run/pipe_router.py b/pipelex/pipe_run/pipe_router.pyindex 5b07d8ec6..7db704620 100644--- a/pipelex/pipe_run/pipe_router.py+++ b/pipelex/pipe_run/pipe_router.py@@ -7,8 +7,8 @@ from pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocolclass PipeRouter(PipeRouterProtocol):- def __init__(self, observer_provider: ObserverProtocol | None = None):- self.observer_provider = observer_provider or ObserverNoOp()+ def __init__(self, observer: ObserverProtocol | None = None):+ self.observer = observer or ObserverNoOp()@overrideasync def _run_pipe_job(diff --git a/pipelex/pipe_run/pipe_router_protocol.py b/pipelex/pipe_run/pipe_router_protocol.pyindex bc98b6788..52d1c5aea 100644--- a/pipelex/pipe_run/pipe_router_protocol.py+++ b/pipelex/pipe_run/pipe_router_protocol.py@@ -6,7 +6,7 @@ from pipelex.pipe_run.pipe_job import PipeJobclass PipeRouterProtocol(Protocol):- observer_provider: ObserverProtocol+ observer: ObserverProtocolasync def _before_run(self,@@ -16,7 +16,7 @@ class PipeRouterProtocol(Protocol):"pipeline_run_id": pipe_job.job_metadata.pipeline_run_id,"pipe_job": pipe_job,}- await self.observer_provider.observe_before_run(payload)+ await self.observer.observe_before_run(payload)async def _after_successful_run(self,@@ -28,7 +28,7 @@ class PipeRouterProtocol(Protocol):"pipe_job": pipe_job,"pipe_output": pipe_output,}- await self.observer_provider.observe_after_successful_run(payload)+ await self.observer.observe_after_successful_run(payload)async def _after_failing_run(self,@@ -40,7 +40,7 @@ class PipeRouterProtocol(Protocol):"pipe_job": pipe_job,"error": error,}- await self.observer_provider.observe_after_failing_run(payload)+ await self.observer.observe_after_failing_run(payload)async def run(self,diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 802515698..2c048c758 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -34,14 +34,10 @@ from pipelex.exceptions import PipelexConfigError, PipelexSetupErrorfrom pipelex.hub import PipelexHub, set_pipelex_hubfrom pipelex.libraries.library_manager_factory import LibraryManagerFactoryfrom pipelex.observer.local_observer import LocalObserver+from pipelex.observer.multi_observer import MultiObserverfrom pipelex.observer.observer_protocol import ObserverProtocolfrom pipelex.pipe_run.pipe_router import PipeRouterfrom pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol-from pipelex.pipeline.activity.activity_manager import ActivityManager-from pipelex.pipeline.activity.activity_manager_protocol import (- ActivityManagerNoOp,- ActivityManagerProtocol,-)from pipelex.pipeline.pipeline_manager import PipelineManagerfrom pipelex.pipeline.track.pipeline_tracker import PipelineTrackerfrom pipelex.pipeline.track.pipeline_tracker_protocol import (@@ -69,19 +65,10 @@ class Pipelex(metaclass=MetaSingleton):def __init__(self,config_dir_path: str = "./pipelex",- # Dependency injection- pipelex_hub: PipelexHub | None = None,config_cls: type[ConfigRoot] | None = None,- class_registry: ClassRegistryAbstract | None = None,- models_manager: ModelManagerAbstract | None = None,- inference_manager: InferenceManager | None = None,- pipeline_manager: PipelineManager | None = None,- pipeline_tracker: PipelineTracker | None = None,- activity_manager: ActivityManagerProtocol | None = None,- reporting_delegate: ReportingProtocol | None = None,) -> None:self.config_dir_path = config_dir_path- self.pipelex_hub = pipelex_hub or PipelexHub()+ self.pipelex_hub = PipelexHub()set_pipelex_hub(self.pipelex_hub)# tools@@ -96,27 +83,12 @@ class Pipelex(metaclass=MetaSingleton):log.verbose("Logs are configured")# tools- self.class_registry = class_registry or ClassRegistry()- self.pipelex_hub.set_class_registry(self.class_registry)- self.kajson_manager = KajsonManager(class_registry=self.class_registry)+ self.class_registry: ClassRegistryAbstract | None = None# cogtself.plugin_manager = PluginManager()self.pipelex_hub.set_plugin_manager(self.plugin_manager)- self.models_manager: ModelManagerAbstract = models_manager or ModelManager()- self.pipelex_hub.set_models_manager(models_manager=self.models_manager)-- self.inference_manager = inference_manager or InferenceManager()- self.pipelex_hub.set_inference_manager(self.inference_manager)-- self.reporting_delegate: ReportingProtocol- if get_config().pipelex.feature_config.is_reporting_enabled:- self.reporting_delegate = reporting_delegate or ReportingManager()- else:- self.reporting_delegate = ReportingNoOp()- self.pipelex_hub.set_report_delegate(self.reporting_delegate)-# pipelex librariesdomain_library = DomainLibrary.make_empty()concept_library = ConceptLibrary.make_empty()@@ -132,26 +104,10 @@ class Pipelex(metaclass=MetaSingleton):)self.pipelex_hub.set_library_manager(library_manager=self.library_manager)- # pipelex pipeline- self.pipeline_tracker: PipelineTrackerProtocol- if pipeline_tracker:- self.pipeline_tracker = pipeline_tracker- elif get_config().pipelex.feature_config.is_pipeline_tracking_enabled:- self.pipeline_tracker = PipelineTracker(tracker_config=get_config().pipelex.tracker_config)- else:- self.pipeline_tracker = PipelineTrackerNoOp()- self.pipelex_hub.set_pipeline_tracker(pipeline_tracker=self.pipeline_tracker)- self.pipeline_manager = pipeline_manager or PipelineManager()- self.pipelex_hub.set_pipeline_manager(pipeline_manager=self.pipeline_manager)+ self.reporting_delegate: ReportingProtocol | None = None- self.activity_manager: ActivityManagerProtocol- if activity_manager:- self.activity_manager = activity_manager- elif get_config().pipelex.feature_config.is_activity_tracking_enabled:- self.activity_manager = ActivityManager()- else:- self.activity_manager = ActivityManagerNoOp()- self.pipelex_hub.set_activity_manager(activity_manager=self.activity_manager)+ # pipeline+ self.pipeline_tracker: PipelineTrackerProtocol | None = Nonelog.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")@@ -183,17 +139,31 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.def setup(self,+ class_registry: ClassRegistryAbstract | None = None,secrets_provider: SecretsProviderAbstract | None = None,+ storage_provider: StorageProviderAbstract | None = None,+ models_manager: ModelManagerAbstract | None = None,+ inference_manager: InferenceManager | None = None,content_generator: ContentGeneratorProtocol | None = None,+ pipeline_manager: PipelineManager | None = None,+ pipeline_tracker: PipelineTracker | None = None,pipe_router: PipeRouterProtocol | None = None,- storage_provider: StorageProviderAbstract | None = None,- observer_provider: ObserverProtocol | None = None,+ reporting_delegate: ReportingProtocol | None = None,+ observers: dict[str, ObserverProtocol] | None = None,):# tools+ self.class_registry = class_registry or ClassRegistry()+ self.pipelex_hub.set_class_registry(self.class_registry)+ self.kajson_manager = KajsonManager(class_registry=self.class_registry)self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())self.pipelex_hub.set_storage_provider(storage_provider)+# cogtself.plugin_manager.setup()++ self.models_manager: ModelManagerAbstract = models_manager or ModelManager()+ self.pipelex_hub.set_models_manager(models_manager=self.models_manager)+try:self.models_manager.setup()except RoutingProfileLibraryNotFoundError as routing_not_found_exc:@@ -238,17 +208,38 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.)raise PipelexSetupError(error_msg) from credentials_excself.pipelex_hub.set_content_generator(content_generator or ContentGenerator())++ self.inference_manager = inference_manager or InferenceManager()+ self.pipelex_hub.set_inference_manager(self.inference_manager)++ # reporting+ if get_config().pipelex.feature_config.is_reporting_enabled:+ self.reporting_delegate = reporting_delegate or ReportingManager()+ else:+ self.reporting_delegate = ReportingNoOp()+ self.pipelex_hub.set_report_delegate(self.reporting_delegate)self.reporting_delegate.setup()++ # pipeline+ if pipeline_tracker:+ self.pipeline_tracker = pipeline_tracker+ elif get_config().pipelex.feature_config.is_pipeline_tracking_enabled:+ self.pipeline_tracker = PipelineTracker(tracker_config=get_config().pipelex.tracker_config)+ else:+ self.pipeline_tracker = PipelineTrackerNoOp()+ self.pipelex_hub.set_pipeline_tracker(pipeline_tracker=self.pipeline_tracker)+ self.pipeline_manager = pipeline_manager or PipelineManager()+ self.pipelex_hub.set_pipeline_manager(pipeline_manager=self.pipeline_manager)+self.class_registry.register_classes(CoreRegistryModels.get_all_models())if runtime_manager.is_unit_testing:log.verbose("Registering test models for unit testing")self.class_registry.register_classes(TestRegistryModels.get_all_models())- self.activity_manager.setup()-- observer_provider = observer_provider or LocalObserver()- self.pipelex_hub.set_observer_provider(observer_provider=observer_provider)- self.pipelex_hub.set_pipe_router(pipe_router or PipeRouter(observer_provider=observer_provider))+ observers = observers or {"local": LocalObserver()}+ multi_observer = MultiObserver(observers=observers)+ self.pipelex_hub.set_observer(observer=multi_observer)+ self.pipelex_hub.set_pipe_router(pipe_router or PipeRouter(observer=multi_observer))# pipelineself.pipeline_tracker.setup()@@ -273,18 +264,20 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.def teardown(self):# pipelexself.pipeline_manager.teardown()- self.pipeline_tracker.teardown()+ if self.pipeline_tracker:+ self.pipeline_tracker.teardown()self.library_manager.teardown()- self.activity_manager.teardown()# cogtself.inference_manager.teardown()- self.reporting_delegate.teardown()+ if self.reporting_delegate:+ self.reporting_delegate.teardown()self.plugin_manager.teardown()# toolsself.kajson_manager.teardown()- self.class_registry.teardown()+ if self.class_registry:+ self.class_registry.teardown()func_registry.teardown()log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} teardown done (except config & logs)")diff --git a/pipelex/pipeline/activity/__init__.py b/pipelex/pipeline/activity/__init__.pydeleted file mode 100644index 8b1378917..000000000--- a/pipelex/pipeline/activity/__init__.py+++ /dev/null@@ -1 +0,0 @@-diff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.pydeleted file mode 100644index 7b5fd002e..000000000--- a/pipelex/pipeline/activity/activity_handler.py+++ /dev/null@@ -1,130 +0,0 @@-import os-from typing import cast--from pipelex import log-from pipelex.config import get_config-from pipelex.core.stuffs.html_content import HtmlContent-from pipelex.core.stuffs.image_content import ImageContent-from pipelex.core.stuffs.list_content import ListContent-from pipelex.core.stuffs.mermaid_content import MermaidContent-from pipelex.core.stuffs.number_content import NumberContent-from pipelex.core.stuffs.structured_content import StructuredContent-from pipelex.core.stuffs.stuff import Stuff-from pipelex.core.stuffs.stuff_content import StuffContent-from pipelex.core.stuffs.text_content import TextContent-from pipelex.pipeline.activity.activity_models import ActivityReport-from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx-from pipelex.tools.misc.file_utils import ensure_path, save_text_to_path-from pipelex.tools.misc.json_utils import save_as_json_to_path---class ActivityHandlerForResultFiles:- def __init__(self, result_dir_path: str):- self.result_dir_path = result_dir_path- self.images_dir_path = os.path.join(result_dir_path, "images")- ensure_path(self.images_dir_path)- img_gen_config = get_config().cogt.img_gen_config- img_gen_param_defaults = img_gen_config.img_gen_param_defaults- self.image_output_format = img_gen_param_defaults.output_format- self.already_handled_stuff: set[str] = set()-- def _generate_stuff_id(self, stuff: Stuff) -> str:- # Use name if available, otherwise just use code- name_part = stuff.stuff_name.replace(" ", "_") if stuff.stuff_name else ""- return f"{name_part}_{stuff.stuff_code}" if name_part else stuff.stuff_code-- def handle_activity(self, activity_report: ActivityReport) -> None:- if isinstance(activity_report.content, Stuff):- the_stuff = activity_report.content- if the_stuff.stuff_code in self.already_handled_stuff:- log.info(f"Already handled stuff: {the_stuff.stuff_name}")- return- self.handle_stuff(stuff=the_stuff)- if code := the_stuff.stuff_code:- self.already_handled_stuff.add(code)- else:- log.error(f"Unhandled activity_report: {activity_report}")-- def handle_stuff(self, stuff: Stuff) -> None:- # Create a directory for this stuff using its code and name- stuff_id = self._generate_stuff_id(stuff)-- # Handle different content types- if isinstance(stuff.content, TextContent):- self._handle_text_content(content=stuff.content, stuff_id=stuff_id)- elif isinstance(stuff.content, NumberContent):- self._handle_number_content(content=stuff.content, stuff_id=stuff_id)- elif isinstance(stuff.content, ImageContent):- self._handle_image_content(content=stuff.content, stuff_id=stuff_id)- elif isinstance(stuff.content, HtmlContent):- self._handle_html_content(content=stuff.content, stuff_id=stuff_id)- elif isinstance(stuff.content, MermaidContent):- self._handle_mermaid_content(content=stuff.content, stuff_id=stuff_id)- elif isinstance(stuff.content, StructuredContent):- self._handle_structured_content(content=stuff.content, stuff_id=stuff_id)- elif isinstance(stuff.content, ListContent):- # TODO: check that all items are StuffContent- self._handle_list_content(content=cast("ListContent[StuffContent]", stuff.content), stuff_id=stuff_id) # pyright: ignore[reportUnknownMemberType]- else:- log.error(f"Unhandled stuff content type: {type(stuff.content)}")-- def _handle_text_content(self, content: TextContent, stuff_id: str) -> None:- stuff_dir = os.path.join(self.result_dir_path, stuff_id)- ensure_path(stuff_dir)-- save_text_to_path(content.text, os.path.join(stuff_dir, f"{stuff_id}.txt"))- # Also save rendered versions- save_text_to_path(content.rendered_html(), os.path.join(stuff_dir, f"{stuff_id}.html"))- save_text_to_path(content.rendered_markdown(), os.path.join(stuff_dir, f"{stuff_id}.md"))-- def _handle_number_content(self, content: NumberContent, stuff_id: str) -> None:- stuff_dir = os.path.join(self.result_dir_path, stuff_id)- ensure_path(stuff_dir)- save_text_to_path(str(content.number), os.path.join(stuff_dir, f"{stuff_id}.txt"))-- def _handle_image_content(self, content: ImageContent, stuff_id: str) -> None:- # Save the image- image_path = os.path.join(self.images_dir_path, f"{stuff_id}.{self.image_output_format}")- if content.url.startswith("http"):- image_bytes: bytes = fetch_file_from_url_httpx(url=content.url, request_timeout=10)- with open(image_path, "wb") as image_file:- image_file.write(image_bytes)- else:- image_path = content.url-- ensure_path(self.images_dir_path)-- def _handle_html_content(self, content: HtmlContent, stuff_id: str) -> None:- stuff_dir = os.path.join(self.result_dir_path, stuff_id)- ensure_path(stuff_dir)- # Save the raw HTML- save_text_to_path(content.inner_html, os.path.join(stuff_dir, f"{stuff_id}.html"))- # Save CSS class- save_text_to_path(content.css_class, os.path.join(stuff_dir, f"{stuff_id}_css.txt"))-- def _handle_mermaid_content(self, content: MermaidContent, stuff_id: str) -> None:- # Save the Mermaid code- save_text_to_path(content.mermaid_code, os.path.join(self.result_dir_path, f"{stuff_id}.mmd"))-- def _handle_structured_content(self, content: StructuredContent, stuff_id: str) -> None:- stuff_dir = os.path.join(self.result_dir_path, stuff_id)- ensure_path(stuff_dir)- # Save the structured content as JSON- save_as_json_to_path(content, os.path.join(stuff_dir, f"{stuff_id}.json"))- # Save rendered versions- save_text_to_path(content.rendered_markdown(), os.path.join(stuff_dir, f"{stuff_id}.md"))-- def _handle_list_content(self, content: ListContent[StuffContent], stuff_id: str) -> None:- stuff_dir = os.path.join(self.result_dir_path, stuff_id)- ensure_path(stuff_dir)- # Save each item in the list- items_dir = os.path.join(stuff_dir, f"{stuff_id}_items")- ensure_path(items_dir)- for idx, item in enumerate(content.items):- item_dir = os.path.join(items_dir, f"item_{idx}")- ensure_path(item_dir)- save_text_to_path(str(item), os.path.join(item_dir, f"{stuff_id}_item_{idx}.txt"))-- # Save rendered versions of the full list- save_text_to_path(content.rendered_markdown(), os.path.join(stuff_dir, f"{stuff_id}.md"))- save_text_to_path(content.rendered_json(), os.path.join(stuff_dir, f"{stuff_id}.json"))diff --git a/pipelex/pipeline/activity/activity_manager.py b/pipelex/pipeline/activity/activity_manager.pydeleted file mode 100644index 1a0a2a044..000000000--- a/pipelex/pipeline/activity/activity_manager.py+++ /dev/null@@ -1,44 +0,0 @@-from pydantic import Field-from typing_extensions import override--from pipelex import log-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol-from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReport-from pipelex.system.exceptions import RootException---class ActivityManagerError(RootException):- pass---class ActivityManager(ActivityManagerProtocol):- def __init__(self) -> None:- self.activity_callbacks: dict[str, ActivityCallback] = Field(default_factory=dict)-- @override- def setup(self) -> None:- self._reset()-- @override- def teardown(self) -> None:- self._reset()- log.verbose("ActivityManager teardown done")-- def _reset(self):- self.activity_callbacks = {}-- @override- def add_activity_callback(self, key: str, callback: ActivityCallback):- if key in self.activity_callbacks:- log.warning(f"Activity callback with key '{key}' already exists")- self.activity_callbacks[key] = callback-- @override- def remove_activity_callback(self, key: str):- self.activity_callbacks.pop(key, None)-- @override- def dispatch_activity(self, activity_report: ActivityReport):- for key, callback in self.activity_callbacks.items():- log.verbose(f"Dispatching activity to callback '{key}'")- callback(activity_report)diff --git a/pipelex/pipeline/activity/activity_manager_protocol.py b/pipelex/pipeline/activity/activity_manager_protocol.pydeleted file mode 100644index a950b9281..000000000--- a/pipelex/pipeline/activity/activity_manager_protocol.py+++ /dev/null@@ -1,44 +0,0 @@-from typing import Protocol--from typing_extensions import override--from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReport---class ActivityManagerProtocol(Protocol):- def setup(self) -> None:- pass-- def teardown(self) -> None:- pass-- def add_activity_callback(self, key: str, callback: ActivityCallback):- pass-- def remove_activity_callback(self, key: str):- pass-- def dispatch_activity(self, activity_report: ActivityReport):- pass---class ActivityManagerNoOp(ActivityManagerProtocol):- @override- def setup(self) -> None:- pass-- @override- def teardown(self) -> None:- pass-- @override- def add_activity_callback(self, key: str, callback: ActivityCallback):- pass-- @override- def remove_activity_callback(self, key: str):- pass-- @override- def dispatch_activity(self, activity_report: ActivityReport):- passdiff --git a/pipelex/pipeline/activity/activity_models.py b/pipelex/pipeline/activity/activity_models.pydeleted file mode 100644index fdb878e74..000000000--- a/pipelex/pipeline/activity/activity_models.py+++ /dev/null@@ -1,14 +0,0 @@-from collections.abc import Callable-from typing import Any--from pydantic import BaseModel--from pipelex.pipeline.job_metadata import JobMetadata---class ActivityReport(BaseModel):- job_metadata: JobMetadata- content: Any---ActivityCallback = Callable[[ActivityReport], None]diff --git a/tests/integration/pipelex/pipes/conftest.py b/tests/integration/pipelex/pipes/conftest.pyindex c8abad753..3e040bc71 100644--- a/tests/integration/pipelex/pipes/conftest.py+++ b/tests/integration/pipelex/pipes/conftest.py@@ -1,41 +1,10 @@import os-from collections.abc import AsyncIterator, Awaitable, Callable+from collections.abc import Awaitable, Callableimport pytest_asyncio-from pytest import FixtureRequestfrom pipelex.core.pipes.pipe_output import PipeOutput-from pipelex.hub import get_activity_manager-from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles-from pipelex.tools.misc.file_utils import get_incremental_directory_path, remove_folderfrom pipelex.tools.misc.json_utils import save_as_json_to_path-from tests.conftest import TEST_OUTPUTS_DIR---@pytest_asyncio.fixture # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]-async def pipe_result_handler(request: FixtureRequest) -> AsyncIterator[tuple[str, ActivityHandlerForResultFiles]]:- """This fixture is used to handle the result of a pipe run in unit tests.- It creates and registers an activity handler to save the activities of the pipe into a specific directory.- It returns a tuple with the result directory path and the activity handler, which enables the- calling test to save whatever it wants into that result directory.- """- # Setup result handler- pipe_code: str = request.node.callspec.params.get("pipe_code", "test_pipe") if hasattr(request.node, "callspec") else "test_pipe" # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType,reportUnknownVariableType]- if not isinstance(pipe_code, str):- msg = f"pipe_code is not a string: {pipe_code}"- raise TypeError(msg)- result_dir_path = get_incremental_directory_path(- base_path=TEST_OUTPUTS_DIR,- base_name=pipe_code,- )- activity_handler = ActivityHandlerForResultFiles(result_dir_path=result_dir_path)- get_activity_manager().add_activity_callback(key="pipelex_unit_test", callback=activity_handler.handle_activity)-- yield result_dir_path, activity_handler-- # Cleanup after test- get_activity_manager().remove_activity_callback(key="pipelex_unit_test")- remove_folder(result_dir_path)@pytest_asyncio.fixture # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.pyindex 8e29823e5..e819a7df6 100644--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py@@ -1,5 +1,3 @@-from typing import Any-import pytestfrom pytest import FixtureRequest@@ -11,7 +9,6 @@ from pipelex.hub import get_pipe_router, get_required_pipefrom pipelex.pipe_run.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_run.pipe_run_params import PipeRunModefrom pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFilesfrom pipelex.pipeline.job_metadata import JobMetadatafrom tests.integration.pipelex.test_data import PipeTestCases@@ -27,15 +24,13 @@ class TestPipeRunningVariants:self,pipe_run_mode: PipeRunMode,request: FixtureRequest,- pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],- save_working_memory: Any,topic: str,stuff: Stuff,pipe_code: str,):log.verbose(stuff, title=f"{topic}: start from '{stuff.stuff_name}', run pipe '{pipe_code}'")working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)- pipe_output = await get_pipe_router().run(+ _ = await get_pipe_router().run(pipe_job=PipeJobFactory.make_pipe_job(pipe=get_required_pipe(pipe_code=pipe_code),pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),@@ -44,17 +39,11 @@ class TestPipeRunningVariants:),)- # Save stuff context- result_dir_path, _ = pipe_result_handler- await save_working_memory(pipe_output, result_dir_path)-@pytest.mark.parametrize(("topic", "pipe_code"), PipeTestCases.NO_INPUT)async def test_pipe_no_input(self,pipe_run_mode: PipeRunMode,request: FixtureRequest,- pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],- save_working_memory: Any,topic: str,pipe_code: str,):@@ -68,10 +57,6 @@ class TestPipeRunningVariants:),)- # Save stuff context- result_dir_path, _ = pipe_result_handler- await save_working_memory(pipe_output, result_dir_path)-stuff = pipe_output.main_stuffpretty_print(stuff, title=f"{topic}: run pipe '{pipe_code}'")pretty_print(stuff.content.rendered_html(), title=f"{topic}: run pipe '{pipe_code}' in html")@@ -82,8 +67,6 @@ class TestPipeRunningVariants:self,pipe_run_mode: PipeRunMode,request: FixtureRequest,- pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],- save_working_memory: Any,topic: str,pipe_code: str,output_multiplicity: VariableMultiplicity | None,@@ -101,10 +84,6 @@ class TestPipeRunningVariants:),)- # Save stuff context- result_dir_path, _ = pipe_result_handler- await save_working_memory(pipe_output, result_dir_path)-stuff = pipe_output.main_stuffpretty_print(stuff, title=f"{topic}: run pipe '{pipe_code}'")pretty_print(stuff.content.rendered_html(), title=f"{topic}: run pipe '{pipe_code}' in html")diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.pyindex a037eee51..aa5f52ac9 100644--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py@@ -75,13 +75,13 @@ class PipeLLMTestCases:inputs={},output="Text",prompt="Generate text",- llm_skill=LLMSkill.LLM_FOR_COMPLEX_REASONING,+ llm_skill=LLMSkill.LLM_TO_WRITE_QUESTIONS,),PipeLLMBlueprint(source=None,type="PipeLLM",description="Generate with preset",- model=LLMSkill.LLM_FOR_COMPLEX_REASONING,+ model=LLMSkill.LLM_TO_WRITE_QUESTIONS,output="Text",prompt="Generate text",),