Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Refactor/cleanup setup and dependency injections (#381)

Louis Choquelade692d48439
diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md
index 5a54470dd..b467128ac 100644
--- a/.github/copilot-instructions.md
+++ b/.github/copilot-instructions.md
@@ -1,6 +1,4 @@
<!-- BEGIN_PIPELEX_RULES -->
-# Pipelex Rules
-
## Guide to write or edit pipelines using the Pipelex language in .plx files
- Always first write your "plan" in natural language, then transcribe it in pipelex.
diff --git a/.windsurfrules.md b/.windsurfrules.md
index 5a54470dd..b467128ac 100644
--- a/.windsurfrules.md
+++ b/.windsurfrules.md
@@ -1,6 +1,4 @@
<!-- BEGIN_PIPELEX_RULES -->
-# Pipelex Rules
-
## Guide to write or edit pipelines using the Pipelex language in .plx files
- Always first write your "plan" in natural language, then transcribe it in pipelex.
diff --git a/AGENTS.md b/AGENTS.md
index 5a54470dd..b467128ac 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -1,6 +1,4 @@
<!-- BEGIN_PIPELEX_RULES -->
-# Pipelex Rules
-
## Guide to write or edit pipelines using the Pipelex language in .plx files
- Always first write your "plan" in natural language, then transcribe it in pipelex.
diff --git a/BLACKBOX_RULES.md b/BLACKBOX_RULES.md
index 5a54470dd..b467128ac 100644
--- a/BLACKBOX_RULES.md
+++ b/BLACKBOX_RULES.md
@@ -1,6 +1,4 @@
<!-- BEGIN_PIPELEX_RULES -->
-# Pipelex Rules
-
## Guide to write or edit pipelines using the Pipelex language in .plx files
- Always first write your "plan" in natural language, then transcribe it in pipelex.
diff --git a/CLAUDE.md b/CLAUDE.md
index 5a54470dd..b467128ac 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -1,6 +1,4 @@
<!-- BEGIN_PIPELEX_RULES -->
-# Pipelex Rules
-
## Guide to write or edit pipelines using the Pipelex language in .plx files
- Always first write your "plan" in natural language, then transcribe it in pipelex.
diff --git a/docs/pages/advanced-customization/observer-provider-injection.md b/docs/pages/advanced-customization/observer-provider-injection.md
index ab71f0d13..3f45bfb5f 100644
--- a/docs/pages/advanced-customization/observer-provider-injection.md
+++ b/docs/pages/advanced-customization/observer-provider-injection.md
@@ -107,7 +107,7 @@ my_observer = MyCustomObserver(config_param="production")
# Inject it into the hub
hub = get_pipelex_hub()
-hub.set_observer_provider(my_observer)
+hub.set_observer(my_observer)
```
### Step 3: Ensure Proper Integration
@@ -132,7 +132,7 @@ local_observer = LocalObserver()
local_observer = LocalObserver(storage_dir="/path/to/custom/dir")
# Inject the observer
-get_pipelex_hub().set_observer_provider(local_observer)
+get_pipelex_hub().set_observer(local_observer)
```
The LocalObserver creates separate JSONL files for each event type:
@@ -209,7 +209,7 @@ def setup_pipelex():
# Setup custom observer
observer = MyCustomObserver()
- get_pipelex_hub().set_observer_provider(observer)
+ get_pipelex_hub().set_observer(observer)
return pipelex_instance
```
diff --git a/pipelex/builder/pipe/pipe_llm_spec.py b/pipelex/builder/pipe/pipe_llm_spec.py
index d0d30ffba..03bb5d864 100644
--- a/pipelex/builder/pipe/pipe_llm_spec.py
+++ b/pipelex/builder/pipe/pipe_llm_spec.py
@@ -21,7 +21,6 @@ class LLMSkill(StrEnum):
LLM_FOR_VISUAL_ANALYSIS = "llm_for_visual_analysis"
LLM_FOR_VISUAL_DESIGN = "llm_for_visual_design"
LLM_FOR_CREATIVE_WRITING = "llm_for_creative_writing"
- LLM_FOR_COMPLEX_REASONING = "llm_for_complex_reasoning"
LLM_TO_REASON_ON_DIAGRAM = "llm_to_reason_on_diagram"
LLM_TO_ANALYZE_DATA = "llm_to_analyze_data"
LLM_TO_CODE = "llm_to_code"
diff --git a/pipelex/hub.py b/pipelex/hub.py
index b5f7d024d..d8a851d33 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -22,7 +22,6 @@ from pipelex.core.pipes.pipe_library_abstract import PipeLibraryAbstract
from pipelex.libraries.library_manager_abstract import LibraryManagerAbstract
from pipelex.observer.observer_protocol import ObserverProtocol
from pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol
-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol
from pipelex.pipeline.pipeline import Pipeline
from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract
from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocol
@@ -65,8 +64,7 @@ class PipelexHub:
# pipeline
self._pipeline_tracker: PipelineTrackerProtocol | None = None
self._pipeline_manager: PipelineManagerAbstract | None = None
- self._activity_manager: ActivityManagerProtocol | None = None
- self._observer_provider: ObserverProtocol | None = None
+ self._observer: ObserverProtocol | None = None
############################################################
# Class methods for singleton management
@@ -155,14 +153,11 @@ class PipelexHub:
def set_pipeline_manager(self, pipeline_manager: PipelineManagerAbstract):
self._pipeline_manager = pipeline_manager
- def set_activity_manager(self, activity_manager: ActivityManagerProtocol):
- self._activity_manager = activity_manager
-
def set_library_manager(self, library_manager: LibraryManagerAbstract):
self._library_manager = library_manager
- def set_observer_provider(self, observer_provider: ObserverProtocol):
- self._observer_provider = observer_provider
+ def set_observer(self, observer: ObserverProtocol):
+ self._observer = observer
############################################################
# Getters
@@ -269,23 +264,17 @@ class PipelexHub:
raise RuntimeError(msg)
return self._pipeline_manager
- def get_activity_manager(self) -> ActivityManagerProtocol:
- if self._activity_manager is None:
- msg = "Activity manager is not set. You must initialize Pipelex first."
- raise RuntimeError(msg)
- return self._activity_manager
-
def get_required_library_manager(self) -> LibraryManagerAbstract:
if self._library_manager is None:
msg = "Library manager is not set. You must initialize Pipelex first."
raise RuntimeError(msg)
return self._library_manager
- def get_observer_provider(self) -> ObserverProtocol:
- if self._observer_provider is None:
+ def get_observer(self) -> ObserverProtocol:
+ if self._observer is None:
msg = "Observer is not set. You must initialize Pipelex first."
raise RuntimeError(msg)
- return self._observer_provider
+ return self._observer
# Shorthand functions for accessing the singleton
@@ -416,10 +405,6 @@ def get_pipeline_manager() -> PipelineManagerAbstract:
return get_pipelex_hub().get_required_pipeline_manager()
-def get_activity_manager() -> ActivityManagerProtocol:
- return get_pipelex_hub().get_activity_manager()
-
-
def get_pipeline(pipeline_run_id: str) -> Pipeline:
return get_pipeline_manager().get_pipeline(pipeline_run_id=pipeline_run_id)
@@ -428,8 +413,8 @@ def get_library_manager() -> LibraryManagerAbstract:
return get_pipelex_hub().get_required_library_manager()
-def get_observer_provider() -> ObserverProtocol:
- return get_pipelex_hub().get_observer_provider()
+def get_observer() -> ObserverProtocol:
+ return get_pipelex_hub().get_observer()
def get_native_concept(native_concept: NativeConceptCode) -> Concept:
diff --git a/pipelex/kit/index.toml b/pipelex/kit/index.toml
index 148f5aa4f..eff4114d4 100644
--- a/pipelex/kit/index.toml
+++ b/pipelex/kit/index.toml
@@ -63,35 +63,30 @@ globs = ["*.plx", "*.toml"]
[agent_rules.targets.agents]
path = "AGENTS.md"
-strategy = "merge"
marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"
marker_end = "<!-- END_PIPELEX_RULES -->"
heading_1 = "# Pipelex Rules"
[agent_rules.targets.claude]
path = "CLAUDE.md"
-strategy = "merge"
marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"
marker_end = "<!-- END_PIPELEX_RULES -->"
heading_1 = "# Pipelex Rules"
[agent_rules.targets.github_copilot]
path = ".github/copilot-instructions.md"
-strategy = "merge"
marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"
marker_end = "<!-- END_PIPELEX_RULES -->"
heading_1 = "# Pipelex Rules"
[agent_rules.targets.windsurf]
path = ".windsurfrules.md"
-strategy = "merge"
marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"
marker_end = "<!-- END_PIPELEX_RULES -->"
heading_1 = "# Pipelex Rules"
[agent_rules.targets.blackbox]
path = "BLACKBOX_RULES.md"
-strategy = "merge"
marker_begin = "<!-- BEGIN_PIPELEX_RULES -->"
marker_end = "<!-- END_PIPELEX_RULES -->"
heading_1 = "# Pipelex Rules"
diff --git a/pipelex/kit/index_models.py b/pipelex/kit/index_models.py
index a74881181..9e910ac4a 100644
--- a/pipelex/kit/index_models.py
+++ b/pipelex/kit/index_models.py
@@ -22,7 +22,6 @@ class Target(ConfigModel):
"""Configuration for a single-file merge target."""
path: str = Field(description="Path to the target file relative to repo root")
- strategy: str = Field(description="Merge strategy (currently only 'merge' supported)")
marker_begin: str = Field(description="Beginning marker for content insertion")
marker_end: str = Field(description="Ending marker for content insertion")
heading_1: str | None = Field(default=None, description="Main title (H1) to add when inserting into empty file or file with no H1 headings")
diff --git a/pipelex/observer/multi_observer.py b/pipelex/observer/multi_observer.py
new file mode 100644
index 000000000..206bb1958
--- /dev/null
+++ b/pipelex/observer/multi_observer.py
@@ -0,0 +1,31 @@
+from typing_extensions import override
+
+from pipelex.observer.observer_protocol import ObserverProtocol, PayloadType
+
+
+class MultiObserver(ObserverProtocol):
+ def __init__(self, observers: dict[str, ObserverProtocol] | None = None) -> None:
+ self.observers: dict[str, ObserverProtocol] = observers or {}
+
+ def add_observer(self, name: str, observer: ObserverProtocol) -> None:
+ """Add an observer with a given name."""
+ self.observers[name] = observer
+
+ def remove_observer(self, name: str) -> None:
+ """Remove an observer by name."""
+ self.observers.pop(name, None)
+
+ @override
+ async def observe_before_run(self, payload: PayloadType) -> None:
+ for observer in self.observers.values():
+ await observer.observe_before_run(payload)
+
+ @override
+ async def observe_after_successful_run(self, payload: PayloadType) -> None:
+ for observer in self.observers.values():
+ await observer.observe_after_successful_run(payload)
+
+ @override
+ async def observe_after_failing_run(self, payload: PayloadType) -> None:
+ for observer in self.observers.values():
+ await observer.observe_after_failing_run(payload)
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index 8db54169b..a49103871 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -8,9 +8,7 @@ from pipelex.core.memory.working_memory import WorkingMemory
from pipelex.core.pipes.pipe_abstract import PipeAbstract
from pipelex.core.pipes.pipe_output import PipeOutput
from pipelex.core.stuffs.text_content import TextContent
-from pipelex.hub import get_activity_manager
from pipelex.pipe_run.pipe_run_params import PipeRunMode, PipeRunParams
-from pipelex.pipeline.activity.activity_models import ActivityReport
from pipelex.pipeline.job_metadata import JobMetadata
PipeOperatorOutputType = TypeVar("PipeOperatorOutputType", bound=PipeOutput)
@@ -70,12 +68,6 @@ class PipeOperator(PipeAbstract, Generic[PipeOperatorOutputType]):
pipe_run_params=pipe_run_params,
output_name=output_name,
)
- get_activity_manager().dispatch_activity(
- activity_report=ActivityReport(
- job_metadata=job_metadata,
- content=pipe_output.main_stuff,
- ),
- )
pipe_run_params.pop_pipe_from_stack(pipe_code=self.code)
diff --git a/pipelex/pipe_run/pipe_router.py b/pipelex/pipe_run/pipe_router.py
index 5b07d8ec6..7db704620 100644
--- a/pipelex/pipe_run/pipe_router.py
+++ b/pipelex/pipe_run/pipe_router.py
@@ -7,8 +7,8 @@ from pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol
class PipeRouter(PipeRouterProtocol):
- def __init__(self, observer_provider: ObserverProtocol | None = None):
- self.observer_provider = observer_provider or ObserverNoOp()
+ def __init__(self, observer: ObserverProtocol | None = None):
+ self.observer = observer or ObserverNoOp()
@override
async def _run_pipe_job(
diff --git a/pipelex/pipe_run/pipe_router_protocol.py b/pipelex/pipe_run/pipe_router_protocol.py
index bc98b6788..52d1c5aea 100644
--- a/pipelex/pipe_run/pipe_router_protocol.py
+++ b/pipelex/pipe_run/pipe_router_protocol.py
@@ -6,7 +6,7 @@ from pipelex.pipe_run.pipe_job import PipeJob
class PipeRouterProtocol(Protocol):
- observer_provider: ObserverProtocol
+ observer: ObserverProtocol
async def _before_run(
self,
@@ -16,7 +16,7 @@ class PipeRouterProtocol(Protocol):
"pipeline_run_id": pipe_job.job_metadata.pipeline_run_id,
"pipe_job": pipe_job,
}
- await self.observer_provider.observe_before_run(payload)
+ await self.observer.observe_before_run(payload)
async def _after_successful_run(
self,
@@ -28,7 +28,7 @@ class PipeRouterProtocol(Protocol):
"pipe_job": pipe_job,
"pipe_output": pipe_output,
}
- await self.observer_provider.observe_after_successful_run(payload)
+ await self.observer.observe_after_successful_run(payload)
async def _after_failing_run(
self,
@@ -40,7 +40,7 @@ class PipeRouterProtocol(Protocol):
"pipe_job": pipe_job,
"error": error,
}
- await self.observer_provider.observe_after_failing_run(payload)
+ await self.observer.observe_after_failing_run(payload)
async def run(
self,
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 802515698..2c048c758 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -34,14 +34,10 @@ from pipelex.exceptions import PipelexConfigError, PipelexSetupError
from pipelex.hub import PipelexHub, set_pipelex_hub
from pipelex.libraries.library_manager_factory import LibraryManagerFactory
from pipelex.observer.local_observer import LocalObserver
+from pipelex.observer.multi_observer import MultiObserver
from pipelex.observer.observer_protocol import ObserverProtocol
from pipelex.pipe_run.pipe_router import PipeRouter
from pipelex.pipe_run.pipe_router_protocol import PipeRouterProtocol
-from pipelex.pipeline.activity.activity_manager import ActivityManager
-from pipelex.pipeline.activity.activity_manager_protocol import (
- ActivityManagerNoOp,
- ActivityManagerProtocol,
-)
from pipelex.pipeline.pipeline_manager import PipelineManager
from pipelex.pipeline.track.pipeline_tracker import PipelineTracker
from pipelex.pipeline.track.pipeline_tracker_protocol import (
@@ -69,19 +65,10 @@ class Pipelex(metaclass=MetaSingleton):
def __init__(
self,
config_dir_path: str = "./pipelex",
- # Dependency injection
- pipelex_hub: PipelexHub | None = None,
config_cls: type[ConfigRoot] | None = None,
- class_registry: ClassRegistryAbstract | None = None,
- models_manager: ModelManagerAbstract | None = None,
- inference_manager: InferenceManager | None = None,
- pipeline_manager: PipelineManager | None = None,
- pipeline_tracker: PipelineTracker | None = None,
- activity_manager: ActivityManagerProtocol | None = None,
- reporting_delegate: ReportingProtocol | None = None,
) -> None:
self.config_dir_path = config_dir_path
- self.pipelex_hub = pipelex_hub or PipelexHub()
+ self.pipelex_hub = PipelexHub()
set_pipelex_hub(self.pipelex_hub)
# tools
@@ -96,27 +83,12 @@ class Pipelex(metaclass=MetaSingleton):
log.verbose("Logs are configured")
# tools
- self.class_registry = class_registry or ClassRegistry()
- self.pipelex_hub.set_class_registry(self.class_registry)
- self.kajson_manager = KajsonManager(class_registry=self.class_registry)
+ self.class_registry: ClassRegistryAbstract | None = None
# cogt
self.plugin_manager = PluginManager()
self.pipelex_hub.set_plugin_manager(self.plugin_manager)
- self.models_manager: ModelManagerAbstract = models_manager or ModelManager()
- self.pipelex_hub.set_models_manager(models_manager=self.models_manager)
-
- self.inference_manager = inference_manager or InferenceManager()
- self.pipelex_hub.set_inference_manager(self.inference_manager)
-
- self.reporting_delegate: ReportingProtocol
- if get_config().pipelex.feature_config.is_reporting_enabled:
- self.reporting_delegate = reporting_delegate or ReportingManager()
- else:
- self.reporting_delegate = ReportingNoOp()
- self.pipelex_hub.set_report_delegate(self.reporting_delegate)
-
# pipelex libraries
domain_library = DomainLibrary.make_empty()
concept_library = ConceptLibrary.make_empty()
@@ -132,26 +104,10 @@ class Pipelex(metaclass=MetaSingleton):
)
self.pipelex_hub.set_library_manager(library_manager=self.library_manager)
- # pipelex pipeline
- self.pipeline_tracker: PipelineTrackerProtocol
- if pipeline_tracker:
- self.pipeline_tracker = pipeline_tracker
- elif get_config().pipelex.feature_config.is_pipeline_tracking_enabled:
- self.pipeline_tracker = PipelineTracker(tracker_config=get_config().pipelex.tracker_config)
- else:
- self.pipeline_tracker = PipelineTrackerNoOp()
- self.pipelex_hub.set_pipeline_tracker(pipeline_tracker=self.pipeline_tracker)
- self.pipeline_manager = pipeline_manager or PipelineManager()
- self.pipelex_hub.set_pipeline_manager(pipeline_manager=self.pipeline_manager)
+ self.reporting_delegate: ReportingProtocol | None = None
- self.activity_manager: ActivityManagerProtocol
- if activity_manager:
- self.activity_manager = activity_manager
- elif get_config().pipelex.feature_config.is_activity_tracking_enabled:
- self.activity_manager = ActivityManager()
- else:
- self.activity_manager = ActivityManagerNoOp()
- self.pipelex_hub.set_activity_manager(activity_manager=self.activity_manager)
+ # pipeline
+ self.pipeline_tracker: PipelineTrackerProtocol | None = None
log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} init done")
@@ -183,17 +139,31 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
def setup(
self,
+ class_registry: ClassRegistryAbstract | None = None,
secrets_provider: SecretsProviderAbstract | None = None,
+ storage_provider: StorageProviderAbstract | None = None,
+ models_manager: ModelManagerAbstract | None = None,
+ inference_manager: InferenceManager | None = None,
content_generator: ContentGeneratorProtocol | None = None,
+ pipeline_manager: PipelineManager | None = None,
+ pipeline_tracker: PipelineTracker | None = None,
pipe_router: PipeRouterProtocol | None = None,
- storage_provider: StorageProviderAbstract | None = None,
- observer_provider: ObserverProtocol | None = None,
+ reporting_delegate: ReportingProtocol | None = None,
+ observers: dict[str, ObserverProtocol] | None = None,
):
# tools
+ self.class_registry = class_registry or ClassRegistry()
+ self.pipelex_hub.set_class_registry(self.class_registry)
+ self.kajson_manager = KajsonManager(class_registry=self.class_registry)
self.pipelex_hub.set_secrets_provider(secrets_provider or EnvSecretsProvider())
self.pipelex_hub.set_storage_provider(storage_provider)
+
# cogt
self.plugin_manager.setup()
+
+ self.models_manager: ModelManagerAbstract = models_manager or ModelManager()
+ self.pipelex_hub.set_models_manager(models_manager=self.models_manager)
+
try:
self.models_manager.setup()
except RoutingProfileLibraryNotFoundError as routing_not_found_exc:
@@ -238,17 +208,38 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
)
raise PipelexSetupError(error_msg) from credentials_exc
self.pipelex_hub.set_content_generator(content_generator or ContentGenerator())
+
+ self.inference_manager = inference_manager or InferenceManager()
+ self.pipelex_hub.set_inference_manager(self.inference_manager)
+
+ # reporting
+ if get_config().pipelex.feature_config.is_reporting_enabled:
+ self.reporting_delegate = reporting_delegate or ReportingManager()
+ else:
+ self.reporting_delegate = ReportingNoOp()
+ self.pipelex_hub.set_report_delegate(self.reporting_delegate)
self.reporting_delegate.setup()
+
+ # pipeline
+ if pipeline_tracker:
+ self.pipeline_tracker = pipeline_tracker
+ elif get_config().pipelex.feature_config.is_pipeline_tracking_enabled:
+ self.pipeline_tracker = PipelineTracker(tracker_config=get_config().pipelex.tracker_config)
+ else:
+ self.pipeline_tracker = PipelineTrackerNoOp()
+ self.pipelex_hub.set_pipeline_tracker(pipeline_tracker=self.pipeline_tracker)
+ self.pipeline_manager = pipeline_manager or PipelineManager()
+ self.pipelex_hub.set_pipeline_manager(pipeline_manager=self.pipeline_manager)
+
self.class_registry.register_classes(CoreRegistryModels.get_all_models())
if runtime_manager.is_unit_testing:
log.verbose("Registering test models for unit testing")
self.class_registry.register_classes(TestRegistryModels.get_all_models())
- self.activity_manager.setup()
-
- observer_provider = observer_provider or LocalObserver()
- self.pipelex_hub.set_observer_provider(observer_provider=observer_provider)
- self.pipelex_hub.set_pipe_router(pipe_router or PipeRouter(observer_provider=observer_provider))
+ observers = observers or {"local": LocalObserver()}
+ multi_observer = MultiObserver(observers=observers)
+ self.pipelex_hub.set_observer(observer=multi_observer)
+ self.pipelex_hub.set_pipe_router(pipe_router or PipeRouter(observer=multi_observer))
# pipeline
self.pipeline_tracker.setup()
@@ -273,18 +264,20 @@ If you need help, drop by our Discord: we're happy to assist: {URLs.discord}.
def teardown(self):
# pipelex
self.pipeline_manager.teardown()
- self.pipeline_tracker.teardown()
+ if self.pipeline_tracker:
+ self.pipeline_tracker.teardown()
self.library_manager.teardown()
- self.activity_manager.teardown()
# cogt
self.inference_manager.teardown()
- self.reporting_delegate.teardown()
+ if self.reporting_delegate:
+ self.reporting_delegate.teardown()
self.plugin_manager.teardown()
# tools
self.kajson_manager.teardown()
- self.class_registry.teardown()
+ if self.class_registry:
+ self.class_registry.teardown()
func_registry.teardown()
log.verbose(f"{PACKAGE_NAME} version {PACKAGE_VERSION} teardown done (except config & logs)")
diff --git a/pipelex/pipeline/activity/__init__.py b/pipelex/pipeline/activity/__init__.py
deleted file mode 100644
index 8b1378917..000000000
--- a/pipelex/pipeline/activity/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/pipelex/pipeline/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.py
deleted file mode 100644
index 7b5fd002e..000000000
--- a/pipelex/pipeline/activity/activity_handler.py
+++ /dev/null
@@ -1,130 +0,0 @@
-import os
-from typing import cast
-
-from pipelex import log
-from pipelex.config import get_config
-from pipelex.core.stuffs.html_content import HtmlContent
-from pipelex.core.stuffs.image_content import ImageContent
-from pipelex.core.stuffs.list_content import ListContent
-from pipelex.core.stuffs.mermaid_content import MermaidContent
-from pipelex.core.stuffs.number_content import NumberContent
-from pipelex.core.stuffs.structured_content import StructuredContent
-from pipelex.core.stuffs.stuff import Stuff
-from pipelex.core.stuffs.stuff_content import StuffContent
-from pipelex.core.stuffs.text_content import TextContent
-from pipelex.pipeline.activity.activity_models import ActivityReport
-from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx
-from pipelex.tools.misc.file_utils import ensure_path, save_text_to_path
-from pipelex.tools.misc.json_utils import save_as_json_to_path
-
-
-class ActivityHandlerForResultFiles:
- def __init__(self, result_dir_path: str):
- self.result_dir_path = result_dir_path
- self.images_dir_path = os.path.join(result_dir_path, "images")
- ensure_path(self.images_dir_path)
- img_gen_config = get_config().cogt.img_gen_config
- img_gen_param_defaults = img_gen_config.img_gen_param_defaults
- self.image_output_format = img_gen_param_defaults.output_format
- self.already_handled_stuff: set[str] = set()
-
- def _generate_stuff_id(self, stuff: Stuff) -> str:
- # Use name if available, otherwise just use code
- name_part = stuff.stuff_name.replace(" ", "_") if stuff.stuff_name else ""
- return f"{name_part}_{stuff.stuff_code}" if name_part else stuff.stuff_code
-
- def handle_activity(self, activity_report: ActivityReport) -> None:
- if isinstance(activity_report.content, Stuff):
- the_stuff = activity_report.content
- if the_stuff.stuff_code in self.already_handled_stuff:
- log.info(f"Already handled stuff: {the_stuff.stuff_name}")
- return
- self.handle_stuff(stuff=the_stuff)
- if code := the_stuff.stuff_code:
- self.already_handled_stuff.add(code)
- else:
- log.error(f"Unhandled activity_report: {activity_report}")
-
- def handle_stuff(self, stuff: Stuff) -> None:
- # Create a directory for this stuff using its code and name
- stuff_id = self._generate_stuff_id(stuff)
-
- # Handle different content types
- if isinstance(stuff.content, TextContent):
- self._handle_text_content(content=stuff.content, stuff_id=stuff_id)
- elif isinstance(stuff.content, NumberContent):
- self._handle_number_content(content=stuff.content, stuff_id=stuff_id)
- elif isinstance(stuff.content, ImageContent):
- self._handle_image_content(content=stuff.content, stuff_id=stuff_id)
- elif isinstance(stuff.content, HtmlContent):
- self._handle_html_content(content=stuff.content, stuff_id=stuff_id)
- elif isinstance(stuff.content, MermaidContent):
- self._handle_mermaid_content(content=stuff.content, stuff_id=stuff_id)
- elif isinstance(stuff.content, StructuredContent):
- self._handle_structured_content(content=stuff.content, stuff_id=stuff_id)
- elif isinstance(stuff.content, ListContent):
- # TODO: check that all items are StuffContent
- self._handle_list_content(content=cast("ListContent[StuffContent]", stuff.content), stuff_id=stuff_id) # pyright: ignore[reportUnknownMemberType]
- else:
- log.error(f"Unhandled stuff content type: {type(stuff.content)}")
-
- def _handle_text_content(self, content: TextContent, stuff_id: str) -> None:
- stuff_dir = os.path.join(self.result_dir_path, stuff_id)
- ensure_path(stuff_dir)
-
- save_text_to_path(content.text, os.path.join(stuff_dir, f"{stuff_id}.txt"))
- # Also save rendered versions
- save_text_to_path(content.rendered_html(), os.path.join(stuff_dir, f"{stuff_id}.html"))
- save_text_to_path(content.rendered_markdown(), os.path.join(stuff_dir, f"{stuff_id}.md"))
-
- def _handle_number_content(self, content: NumberContent, stuff_id: str) -> None:
- stuff_dir = os.path.join(self.result_dir_path, stuff_id)
- ensure_path(stuff_dir)
- save_text_to_path(str(content.number), os.path.join(stuff_dir, f"{stuff_id}.txt"))
-
- def _handle_image_content(self, content: ImageContent, stuff_id: str) -> None:
- # Save the image
- image_path = os.path.join(self.images_dir_path, f"{stuff_id}.{self.image_output_format}")
- if content.url.startswith("http"):
- image_bytes: bytes = fetch_file_from_url_httpx(url=content.url, request_timeout=10)
- with open(image_path, "wb") as image_file:
- image_file.write(image_bytes)
- else:
- image_path = content.url
-
- ensure_path(self.images_dir_path)
-
- def _handle_html_content(self, content: HtmlContent, stuff_id: str) -> None:
- stuff_dir = os.path.join(self.result_dir_path, stuff_id)
- ensure_path(stuff_dir)
- # Save the raw HTML
- save_text_to_path(content.inner_html, os.path.join(stuff_dir, f"{stuff_id}.html"))
- # Save CSS class
- save_text_to_path(content.css_class, os.path.join(stuff_dir, f"{stuff_id}_css.txt"))
-
- def _handle_mermaid_content(self, content: MermaidContent, stuff_id: str) -> None:
- # Save the Mermaid code
- save_text_to_path(content.mermaid_code, os.path.join(self.result_dir_path, f"{stuff_id}.mmd"))
-
- def _handle_structured_content(self, content: StructuredContent, stuff_id: str) -> None:
- stuff_dir = os.path.join(self.result_dir_path, stuff_id)
- ensure_path(stuff_dir)
- # Save the structured content as JSON
- save_as_json_to_path(content, os.path.join(stuff_dir, f"{stuff_id}.json"))
- # Save rendered versions
- save_text_to_path(content.rendered_markdown(), os.path.join(stuff_dir, f"{stuff_id}.md"))
-
- def _handle_list_content(self, content: ListContent[StuffContent], stuff_id: str) -> None:
- stuff_dir = os.path.join(self.result_dir_path, stuff_id)
- ensure_path(stuff_dir)
- # Save each item in the list
- items_dir = os.path.join(stuff_dir, f"{stuff_id}_items")
- ensure_path(items_dir)
- for idx, item in enumerate(content.items):
- item_dir = os.path.join(items_dir, f"item_{idx}")
- ensure_path(item_dir)
- save_text_to_path(str(item), os.path.join(item_dir, f"{stuff_id}_item_{idx}.txt"))
-
- # Save rendered versions of the full list
- save_text_to_path(content.rendered_markdown(), os.path.join(stuff_dir, f"{stuff_id}.md"))
- save_text_to_path(content.rendered_json(), os.path.join(stuff_dir, f"{stuff_id}.json"))
diff --git a/pipelex/pipeline/activity/activity_manager.py b/pipelex/pipeline/activity/activity_manager.py
deleted file mode 100644
index 1a0a2a044..000000000
--- a/pipelex/pipeline/activity/activity_manager.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from pydantic import Field
-from typing_extensions import override
-
-from pipelex import log
-from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol
-from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReport
-from pipelex.system.exceptions import RootException
-
-
-class ActivityManagerError(RootException):
- pass
-
-
-class ActivityManager(ActivityManagerProtocol):
- def __init__(self) -> None:
- self.activity_callbacks: dict[str, ActivityCallback] = Field(default_factory=dict)
-
- @override
- def setup(self) -> None:
- self._reset()
-
- @override
- def teardown(self) -> None:
- self._reset()
- log.verbose("ActivityManager teardown done")
-
- def _reset(self):
- self.activity_callbacks = {}
-
- @override
- def add_activity_callback(self, key: str, callback: ActivityCallback):
- if key in self.activity_callbacks:
- log.warning(f"Activity callback with key '{key}' already exists")
- self.activity_callbacks[key] = callback
-
- @override
- def remove_activity_callback(self, key: str):
- self.activity_callbacks.pop(key, None)
-
- @override
- def dispatch_activity(self, activity_report: ActivityReport):
- for key, callback in self.activity_callbacks.items():
- log.verbose(f"Dispatching activity to callback '{key}'")
- callback(activity_report)
diff --git a/pipelex/pipeline/activity/activity_manager_protocol.py b/pipelex/pipeline/activity/activity_manager_protocol.py
deleted file mode 100644
index a950b9281..000000000
--- a/pipelex/pipeline/activity/activity_manager_protocol.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from typing import Protocol
-
-from typing_extensions import override
-
-from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReport
-
-
-class ActivityManagerProtocol(Protocol):
- def setup(self) -> None:
- pass
-
- def teardown(self) -> None:
- pass
-
- def add_activity_callback(self, key: str, callback: ActivityCallback):
- pass
-
- def remove_activity_callback(self, key: str):
- pass
-
- def dispatch_activity(self, activity_report: ActivityReport):
- pass
-
-
-class ActivityManagerNoOp(ActivityManagerProtocol):
- @override
- def setup(self) -> None:
- pass
-
- @override
- def teardown(self) -> None:
- pass
-
- @override
- def add_activity_callback(self, key: str, callback: ActivityCallback):
- pass
-
- @override
- def remove_activity_callback(self, key: str):
- pass
-
- @override
- def dispatch_activity(self, activity_report: ActivityReport):
- pass
diff --git a/pipelex/pipeline/activity/activity_models.py b/pipelex/pipeline/activity/activity_models.py
deleted file mode 100644
index fdb878e74..000000000
--- a/pipelex/pipeline/activity/activity_models.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from collections.abc import Callable
-from typing import Any
-
-from pydantic import BaseModel
-
-from pipelex.pipeline.job_metadata import JobMetadata
-
-
-class ActivityReport(BaseModel):
- job_metadata: JobMetadata
- content: Any
-
-
-ActivityCallback = Callable[[ActivityReport], None]
diff --git a/tests/integration/pipelex/pipes/conftest.py b/tests/integration/pipelex/pipes/conftest.py
index c8abad753..3e040bc71 100644
--- a/tests/integration/pipelex/pipes/conftest.py
+++ b/tests/integration/pipelex/pipes/conftest.py
@@ -1,41 +1,10 @@
import os
-from collections.abc import AsyncIterator, Awaitable, Callable
+from collections.abc import Awaitable, Callable
import pytest_asyncio
-from pytest import FixtureRequest
from pipelex.core.pipes.pipe_output import PipeOutput
-from pipelex.hub import get_activity_manager
-from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
-from pipelex.tools.misc.file_utils import get_incremental_directory_path, remove_folder
from pipelex.tools.misc.json_utils import save_as_json_to_path
-from tests.conftest import TEST_OUTPUTS_DIR
-
-
-@pytest_asyncio.fixture # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]
-async def pipe_result_handler(request: FixtureRequest) -> AsyncIterator[tuple[str, ActivityHandlerForResultFiles]]:
- """This fixture is used to handle the result of a pipe run in unit tests.
- It creates and registers an activity handler to save the activities of the pipe into a specific directory.
- It returns a tuple with the result directory path and the activity handler, which enables the
- calling test to save whatever it wants into that result directory.
- """
- # Setup result handler
- pipe_code: str = request.node.callspec.params.get("pipe_code", "test_pipe") if hasattr(request.node, "callspec") else "test_pipe" # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType,reportUnknownVariableType]
- if not isinstance(pipe_code, str):
- msg = f"pipe_code is not a string: {pipe_code}"
- raise TypeError(msg)
- result_dir_path = get_incremental_directory_path(
- base_path=TEST_OUTPUTS_DIR,
- base_name=pipe_code,
- )
- activity_handler = ActivityHandlerForResultFiles(result_dir_path=result_dir_path)
- get_activity_manager().add_activity_callback(key="pipelex_unit_test", callback=activity_handler.handle_activity)
-
- yield result_dir_path, activity_handler
-
- # Cleanup after test
- get_activity_manager().remove_activity_callback(key="pipelex_unit_test")
- remove_folder(result_dir_path)
@pytest_asyncio.fixture # pyright: ignore[reportUntypedFunctionDecorator, reportUnknownMemberType]
diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
index 8e29823e5..e819a7df6 100644
--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py
+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
@@ -1,5 +1,3 @@
-from typing import Any
-
import pytest
from pytest import FixtureRequest
@@ -11,7 +9,6 @@ from pipelex.hub import get_pipe_router, get_required_pipe
from pipelex.pipe_run.pipe_job_factory import PipeJobFactory
from pipelex.pipe_run.pipe_run_params import PipeRunMode
from pipelex.pipe_run.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
from pipelex.pipeline.job_metadata import JobMetadata
from tests.integration.pipelex.test_data import PipeTestCases
@@ -27,15 +24,13 @@ class TestPipeRunningVariants:
self,
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
- pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],
- save_working_memory: Any,
topic: str,
stuff: Stuff,
pipe_code: str,
):
log.verbose(stuff, title=f"{topic}: start from '{stuff.stuff_name}', run pipe '{pipe_code}'")
working_memory = WorkingMemoryFactory.make_from_single_stuff(stuff=stuff)
- pipe_output = await get_pipe_router().run(
+ _ = await get_pipe_router().run(
pipe_job=PipeJobFactory.make_pipe_job(
pipe=get_required_pipe(pipe_code=pipe_code),
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
@@ -44,17 +39,11 @@ class TestPipeRunningVariants:
),
)
- # Save stuff context
- result_dir_path, _ = pipe_result_handler
- await save_working_memory(pipe_output, result_dir_path)
-
@pytest.mark.parametrize(("topic", "pipe_code"), PipeTestCases.NO_INPUT)
async def test_pipe_no_input(
self,
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
- pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],
- save_working_memory: Any,
topic: str,
pipe_code: str,
):
@@ -68,10 +57,6 @@ class TestPipeRunningVariants:
),
)
- # Save stuff context
- result_dir_path, _ = pipe_result_handler
- await save_working_memory(pipe_output, result_dir_path)
-
stuff = pipe_output.main_stuff
pretty_print(stuff, title=f"{topic}: run pipe '{pipe_code}'")
pretty_print(stuff.content.rendered_html(), title=f"{topic}: run pipe '{pipe_code}' in html")
@@ -82,8 +67,6 @@ class TestPipeRunningVariants:
self,
pipe_run_mode: PipeRunMode,
request: FixtureRequest,
- pipe_result_handler: tuple[str, ActivityHandlerForResultFiles],
- save_working_memory: Any,
topic: str,
pipe_code: str,
output_multiplicity: VariableMultiplicity | None,
@@ -101,10 +84,6 @@ class TestPipeRunningVariants:
),
)
- # Save stuff context
- result_dir_path, _ = pipe_result_handler
- await save_working_memory(pipe_output, result_dir_path)
-
stuff = pipe_output.main_stuff
pretty_print(stuff, title=f"{topic}: run pipe '{pipe_code}'")
pretty_print(stuff.content.rendered_html(), title=f"{topic}: run pipe '{pipe_code}' in html")
diff --git a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py
index a037eee51..aa5f52ac9 100644
--- a/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py
+++ b/tests/unit/pipelex/builder/pipe/pipe_operator/pipe_llm/test_data.py
@@ -75,13 +75,13 @@ class PipeLLMTestCases:
inputs={},
output="Text",
prompt="Generate text",
- llm_skill=LLMSkill.LLM_FOR_COMPLEX_REASONING,
+ llm_skill=LLMSkill.LLM_TO_WRITE_QUESTIONS,
),
PipeLLMBlueprint(
source=None,
type="PipeLLM",
description="Generate with preset",
- model=LLMSkill.LLM_FOR_COMPLEX_REASONING,
+ model=LLMSkill.LLM_TO_WRITE_QUESTIONS,
output="Text",
prompt="Generate text",
),