Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
Refactor/rename pipeline (#45)
Louis Choquelfddda669946d
diff --git a/Makefile b/Makefileindex 52b79909a..707873169 100644--- a/Makefile+++ b/Makefile@@ -65,10 +65,12 @@ make init - Run `pipelex init libraries` and `pipelex init cmake codex-tests - Run tests for Codex (exit on first failure) (no inference, no codex_disabled)make gha-tests - Run tests for github actions (exit on first failure) (no inference, no gha_disabled)make test - Run unit tests (no inference)+make test-xdist - Run unit tests with xdist (no inference)+make t - Shorthand -> test-xdistmake test-quiet - Run unit tests without prints (no inference)make tq - Shorthand -> test-quietmake test-with-prints - Run tests with prints (no inference)-make t - Shorthand -> test-with-prints+make tp - Shorthand -> test-with-printsmake test-inference - Run unit tests only for inference (with prints)make ti - Shorthand -> test-inferencemake test-ocr - Run unit tests only for ocr (with prints)@@ -90,7 +92,7 @@ export HELPall help env lock install update build \format lint pyright mypy \cleanderived cleanenv cleanlibraries cleanall \- test test-quiet tq test-with-prints t test-inference ti \+ test test-xdist t test-quiet tq test-with-prints tp test-inference ti \test-imgg tg test-ocr to codex-tests gha-tests \run-all-tests run-manual-trigger-gha-tests run-gha_disabled-tests \validate v check c cc \@@ -227,18 +229,30 @@ test: env$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")@echo "• Running unit tests"@if [ -n "$(TEST)" ]; then \- $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \else \- $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \fi+test-xdist: env+ $(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")+ @echo "• Running unit tests"+ @if [ -n "$(TEST)" ]; then \+ $(VENV_PYTEST) -n auto -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \+ else \+ $(VENV_PYTEST) -n auto -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \+ fi++t: test-xdist+ @echo "> done: t = test-xdist"+test-quiet: env$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")@echo "• Running unit tests"@if [ -n "$(TEST)" ]; then \- $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \else \- $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \fitq: test-quiet@@ -248,20 +262,20 @@ test-with-prints: env$(call PRINT_TITLE,"Unit testing with prints and our rich logs")@echo "• Running unit tests"@if [ -n "$(TEST)" ]; then \- $(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \else \- $(VENV_PYTEST) -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \fi-t: test-with-prints- @echo "> done: t = test-with-prints"+tp: test-with-prints+ @echo "> done: tp = test-with-prints"test-inference: env$(call PRINT_TITLE,"Unit testing")@if [ -n "$(TEST)" ]; then \- $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \else \- $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \fiti: test-inference@@ -270,9 +284,9 @@ ti: test-inferencetest-ocr: env$(call PRINT_TITLE,"Unit testing ocr")@if [ -n "$(TEST)" ]; then \- $(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \else \- $(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \fito: test-ocr@@ -281,9 +295,9 @@ to: test-ocrtest-imgg: env$(call PRINT_TITLE,"Unit testing")@if [ -n "$(TEST)" ]; then \- $(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \else \- $(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \+ $(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \fitg: test-imggdiff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.pyindex 686fbff66..a02d96bb6 100644--- a/pipelex/cogt/content_generation/assignment_models.py+++ b/pipelex/cogt/content_generation/assignment_models.py@@ -16,7 +16,7 @@ from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstracfrom pipelex.cogt.ocr.ocr_handle import OcrHandlefrom pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategoryfrom pipelex.tools.templating.templating_models import PromptingStylediff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.pyindex 5d1486cd8..89f3a2398 100644--- a/pipelex/cogt/content_generation/content_generator.py+++ b/pipelex/cogt/content_generation/content_generator.py@@ -30,7 +30,7 @@ from pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParamsfrom pipelex.cogt.ocr.ocr_output import OcrOutputfrom pipelex.config import get_config-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategoryfrom pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVardiff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.pyindex f14f6a902..034badbd7 100644--- a/pipelex/cogt/content_generation/content_generator_protocol.py+++ b/pipelex/cogt/content_generation/content_generator_protocol.py@@ -12,7 +12,7 @@ from pipelex.cogt.ocr.ocr_handle import OcrHandlefrom pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParamsfrom pipelex.cogt.ocr.ocr_output import OcrOutput-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategoryfrom pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVardiff --git a/pipelex/cogt/imgg/imgg_job_factory.py b/pipelex/cogt/imgg/imgg_job_factory.pyindex f416b1725..91fbb8587 100644--- a/pipelex/cogt/imgg/imgg_job_factory.py+++ b/pipelex/cogt/imgg/imgg_job_factory.py@@ -4,7 +4,7 @@ from pipelex.cogt.imgg.imgg_job import ImggJobfrom pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobReportfrom pipelex.cogt.imgg.imgg_prompt import ImggPromptfrom pipelex.config import get_config-from pipelex.mission.job_metadata import JobCategory, JobMetadata+from pipelex.pipeline.job_metadata import JobCategory, JobMetadataclass ImggJobFactory:diff --git a/pipelex/cogt/imgg/imgg_worker_abstract.py b/pipelex/cogt/imgg/imgg_worker_abstract.pyindex 65dc51c41..384afec00 100644--- a/pipelex/cogt/imgg/imgg_worker_abstract.py+++ b/pipelex/cogt/imgg/imgg_worker_abstract.py@@ -10,7 +10,7 @@ from pipelex.cogt.imgg.imgg_engine import ImggEnginefrom pipelex.cogt.imgg.imgg_job import ImggJobfrom pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegatefrom pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract-from pipelex.mission.job_metadata import UnitJobId+from pipelex.pipeline.job_metadata import UnitJobIdF = TypeVar("F", bound=Callable[..., Awaitable[Any]])diff --git a/pipelex/cogt/inference/cost_registry.py b/pipelex/cogt/inference/cost_registry.pyindex 4e024fa10..c8fc821cc 100644--- a/pipelex/cogt/inference/cost_registry.py+++ b/pipelex/cogt/inference/cost_registry.py@@ -28,13 +28,13 @@ class CostRegistry(RootModel[CostRegistryRoot]):@classmethoddef generate_report(cls,- mission_id: str,+ pipeline_run_id: str,llm_tokens_usages: List[LLMTokensUsage],unit_scale: float,cost_report_file_path: Optional[str] = None,):if not llm_tokens_usages:- log.warning(f"No report to generate for mission '{mission_id}'")+ log.warning(f"No report to generate for pipeline '{pipeline_run_id}'")returncost_registry = CostRegistry()for llm_tokens_usage in llm_tokens_usages:@@ -77,7 +77,7 @@ class CostRegistry(RootModel[CostRegistryRoot]):console = Console()title = "Costs by LLM model"- title += f" for mission '{mission_id}'"+ title += f" for pipeline '{pipeline_run_id}'"table = Table(title=title, box=box.ROUNDED)scale_str: strdiff --git a/pipelex/cogt/inference/inference_job_abstract.py b/pipelex/cogt/inference/inference_job_abstract.pyindex 9b99655d8..da5dd2b2f 100644--- a/pipelex/cogt/inference/inference_job_abstract.py+++ b/pipelex/cogt/inference/inference_job_abstract.py@@ -2,7 +2,7 @@ from abc import ABC, abstractmethodfrom pydantic import BaseModel-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadataclass InferenceJobAbstract(ABC, BaseModel):diff --git a/pipelex/cogt/inference/inference_report_delegate.py b/pipelex/cogt/inference/inference_report_delegate.pyindex a69bbfbbd..8d5d9bf09 100644--- a/pipelex/cogt/inference/inference_report_delegate.py+++ b/pipelex/cogt/inference/inference_report_delegate.py@@ -4,10 +4,10 @@ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstractclass InferenceReportDelegate(Protocol):- def open_registry(self, mission_id: str): ...+ def open_registry(self, pipeline_run_id: str): ...def report_inference_job(self, inference_job: InferenceJobAbstract): ...- def generate_report(self, mission_id: Optional[str] = None): ...+ def generate_report(self, pipeline_run_id: Optional[str] = None): ...- def close_registry(self, mission_id: str): ...+ def close_registry(self, pipeline_run_id: str): ...diff --git a/pipelex/cogt/inference/inference_report_manager.py b/pipelex/cogt/inference/inference_report_manager.pyindex 8457c568f..02f766cc9 100644--- a/pipelex/cogt/inference/inference_report_manager.py+++ b/pipelex/cogt/inference/inference_report_manager.py@@ -11,7 +11,7 @@ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstractfrom pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegatefrom pipelex.cogt.llm.llm_job import LLMJobfrom pipelex.cogt.llm.llm_report import LLMTokenCostReport, LLMTokensUsage-from pipelex.mission.mission_models import SpecialMissionId+from pipelex.pipeline.pipeline_models import SpecialPipelineIdfrom pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_pathLLMUsageRegistryRoot = List[LLMTokensUsage]@@ -38,7 +38,7 @@ class InferenceReportManager(InferenceReportDelegate):def setup(self):self._usage_registries.clear()- self._usage_registries[SpecialMissionId.UNTITLED] = UsageRegistry()+ self._usage_registries[SpecialPipelineId.UNTITLED] = UsageRegistry()def teardown(self):self._usage_registries.clear()@@ -47,10 +47,10 @@ class InferenceReportManager(InferenceReportDelegate):# Private methods############################################################- def _get_registry(self, mission_id: str) -> UsageRegistry:- if mission_id not in self._usage_registries:- raise InferenceReportManagerError(f"Registry for mission '{mission_id}' does not exist")- return self._usage_registries[mission_id]+ def _get_registry(self, pipeline_run_id: str) -> UsageRegistry:+ if pipeline_run_id not in self._usage_registries:+ raise InferenceReportManagerError(f"Registry for pipeline '{pipeline_run_id}' does not exist")+ return self._usage_registries[pipeline_run_id]def _report_llm_job(self, llm_job: LLMJob):llm_tokens_usage = llm_job.job_report.llm_tokens_usage@@ -64,8 +64,8 @@ class InferenceReportManager(InferenceReportDelegate):if self._report_config.is_log_costs_to_console:llm_token_cost_report = CostRegistry.complete_cost_report(llm_tokens_usage=llm_tokens_usage)- mission_id = llm_job.job_metadata.mission_id- self._get_registry(mission_id).add_tokens_usage(llm_tokens_usage)+ pipeline_run_id = llm_job.job_metadata.pipeline_run_id+ self._get_registry(pipeline_run_id).add_tokens_usage(llm_tokens_usage)if self._report_config.is_log_costs_to_console:log.verbose(llm_token_cost_report, title="Token Cost report")@@ -75,10 +75,10 @@ class InferenceReportManager(InferenceReportDelegate):############################################################@override- def open_registry(self, mission_id: str):- if mission_id in self._usage_registries:- raise InferenceReportManagerError(f"Registry for mission '{mission_id}' already exists")- self._usage_registries[mission_id] = UsageRegistry()+ def open_registry(self, pipeline_run_id: str):+ if pipeline_run_id in self._usage_registries:+ raise InferenceReportManagerError(f"Registry for pipeline '{pipeline_run_id}' already exists")+ self._usage_registries[pipeline_run_id] = UsageRegistry()@overridedef report_inference_job(self, inference_job: InferenceJobAbstract):@@ -91,8 +91,8 @@ class InferenceReportManager(InferenceReportDelegate):self._report_llm_job(llm_job=llm_job)@override- def generate_report(self, mission_id: Optional[str] = None):- mission_id = mission_id or SpecialMissionId.UNTITLED+ def generate_report(self, pipeline_run_id: Optional[str] = None):+ pipeline_run_id = pipeline_run_id or SpecialPipelineId.UNTITLEDcost_report_file_path: Optional[str] = Noneif self._report_config.is_generate_cost_report_file_enabled:ensure_path(self._report_config.cost_report_dir_path)@@ -102,14 +102,14 @@ class InferenceReportManager(InferenceReportDelegate):extension=self._report_config.cost_report_extension,)- registry = self._get_registry(mission_id)+ registry = self._get_registry(pipeline_run_id)CostRegistry.generate_report(- mission_id=mission_id,+ pipeline_run_id=pipeline_run_id,llm_tokens_usages=registry.get_current_tokens_usage(),unit_scale=self._report_config.cost_report_unit_scale,cost_report_file_path=cost_report_file_path,)@override- def close_registry(self, mission_id: str):- self._usage_registries.pop(mission_id)+ def close_registry(self, pipeline_run_id: str):+ self._usage_registries.pop(pipeline_run_id)diff --git a/pipelex/cogt/llm/llm_job_factory.py b/pipelex/cogt/llm/llm_job_factory.pyindex e6c9810ff..f96a99865 100644--- a/pipelex/cogt/llm/llm_job_factory.py+++ b/pipelex/cogt/llm/llm_job_factory.py@@ -8,7 +8,7 @@ from pipelex.cogt.llm.llm_prompt import LLMPromptfrom pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstractfrom pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplatefrom pipelex.config import get_config-from pipelex.mission.job_metadata import JobCategory, JobMetadata+from pipelex.pipeline.job_metadata import JobCategory, JobMetadataclass LLMJobFactory:diff --git a/pipelex/cogt/llm/llm_report.py b/pipelex/cogt/llm/llm_report.pyindex f15469888..b373086cb 100644--- a/pipelex/cogt/llm/llm_report.py+++ b/pipelex/cogt/llm/llm_report.py@@ -7,7 +7,7 @@ from pipelex.cogt.llm.llm_models.llm_engine import LLMEnginefrom pipelex.cogt.llm.llm_models.llm_family import LLMFamilyfrom pipelex.cogt.llm.llm_models.llm_platform import LLMPlatformfrom pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategory, TokenCostsByCategoryDict-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.types import StrEnumdiff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.pyindex 7822ae72e..a9ed94be4 100644--- a/pipelex/cogt/llm/llm_worker_abstract.py+++ b/pipelex/cogt/llm/llm_worker_abstract.py@@ -9,7 +9,7 @@ from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstfrom pipelex.cogt.llm.llm_job import LLMJobfrom pipelex.cogt.llm.llm_models.llm_engine import LLMEnginefrom pipelex.cogt.llm.structured_output import StructureMethod-from pipelex.mission.job_metadata import UnitJobId+from pipelex.pipeline.job_metadata import UnitJobIdfrom pipelex.tools.typing.pydantic_utils import BaseModelTypeVarfrom pipelex.types import StrEnumdiff --git a/pipelex/cogt/ocr/ocr_job_factory.py b/pipelex/cogt/ocr/ocr_job_factory.pyindex b6c2e4ce5..9220dd05a 100644--- a/pipelex/cogt/ocr/ocr_job_factory.py+++ b/pipelex/cogt/ocr/ocr_job_factory.py@@ -4,7 +4,7 @@ from pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job import OcrJobfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReportfrom pipelex.config import get_config-from pipelex.mission.job_metadata import JobCategory, JobMetadata+from pipelex.pipeline.job_metadata import JobCategory, JobMetadataclass OcrJobFactory:diff --git a/pipelex/cogt/ocr/ocr_worker_abstract.py b/pipelex/cogt/ocr/ocr_worker_abstract.pyindex 982520cd1..6d5df0f18 100644--- a/pipelex/cogt/ocr/ocr_worker_abstract.py+++ b/pipelex/cogt/ocr/ocr_worker_abstract.py@@ -10,7 +10,7 @@ from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstfrom pipelex.cogt.ocr.ocr_engine import OcrEnginefrom pipelex.cogt.ocr.ocr_job import OcrJobfrom pipelex.cogt.ocr.ocr_output import OcrOutput-from pipelex.mission.job_metadata import UnitJobId+from pipelex.pipeline.job_metadata import UnitJobIdF = TypeVar("F", bound=Callable[..., Awaitable[Any]])diff --git a/pipelex/config.py b/pipelex/config.pyindex d114b8804..7ecaa0fc6 100644--- a/pipelex/config.py+++ b/pipelex/config.py@@ -7,7 +7,7 @@ from pipelex.cogt.llm.llm_models.llm_prompting_target import LLMPromptingTargetfrom pipelex.exceptions import PipelexErrorfrom pipelex.hub import get_required_configfrom pipelex.libraries.library_config import LibraryConfig-from pipelex.mission.track.tracker_config import TrackerConfig+from pipelex.pipeline.track.tracker_config import TrackerConfigfrom pipelex.plugins.plugins_config import PluginsConfigfrom pipelex.tools.aws.aws_config import AwsConfigfrom pipelex.tools.config.models import ConfigModel, ConfigRoot@@ -44,7 +44,7 @@ class PromptingConfig(ConfigModel):class FeatureConfig(ConfigModel):- is_mission_tracking_enabled: bool+ is_pipeline_tracking_enabled: boolis_activity_tracking_enabled: booldiff --git a/pipelex/core/pipe_abstract.py b/pipelex/core/pipe_abstract.pyindex 51a01e9e2..df3d60069 100644--- a/pipelex/core/pipe_abstract.py+++ b/pipelex/core/pipe_abstract.py@@ -7,7 +7,7 @@ from pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeStackOverflowError-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadataclass PipeAbstract(ABC, BaseModel):diff --git a/pipelex/exceptions.py b/pipelex/exceptions.pyindex be6b4ec3f..715f3b6a1 100644--- a/pipelex/exceptions.py+++ b/pipelex/exceptions.py@@ -167,5 +167,5 @@ class ConceptDomainError(ConceptError):pass-class MissionManagerNotFoundError(PipelexError):+class PipelineManagerNotFoundError(PipelexError):passdiff --git a/pipelex/hub.py b/pipelex/hub.pyindex 5c0905e98..95bf99731 100644--- a/pipelex/hub.py+++ b/pipelex/hub.py@@ -18,11 +18,11 @@ from pipelex.core.domain import Domainfrom pipelex.core.domain_provider_abstract import DomainProviderAbstractfrom pipelex.core.pipe_abstract import PipeAbstractfrom pipelex.core.pipe_provider_abstract import PipeProviderAbstract-from pipelex.mission.activity.activity_manager_protocol import ActivityManagerProtocol-from pipelex.mission.mission import Mission-from pipelex.mission.mission_manager_abstract import MissionManagerAbstract-from pipelex.mission.track.mission_tracker_protocol import MissionTrackerProtocolfrom pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol+from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol+from pipelex.pipeline.pipeline import Pipeline+from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract+from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocolfrom pipelex.tools.config.manager import config_managerfrom pipelex.tools.config.models import ConfigRootfrom pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract@@ -57,9 +57,9 @@ class PipelexHub:self._pipe_provider: Optional[PipeProviderAbstract] = Noneself._pipe_router: Optional[PipeRouterProtocol] = None- # mission- self._mission_tracker: Optional[MissionTrackerProtocol] = None- self._mission_manager: Optional[MissionManagerAbstract] = None+ # pipeline+ self._pipeline_tracker: Optional[PipelineTrackerProtocol] = None+ self._pipeline_manager: Optional[PipelineManagerAbstract] = Noneself._activity_manager: Optional[ActivityManagerProtocol] = None############################################################@@ -146,11 +146,11 @@ class PipelexHub:def set_pipe_router(self, pipe_router: PipeRouterProtocol):self._pipe_router = pipe_router- def set_mission_tracker(self, mission_tracker: MissionTrackerProtocol):- self._mission_tracker = mission_tracker+ def set_pipeline_tracker(self, pipeline_tracker: PipelineTrackerProtocol):+ self._pipeline_tracker = pipeline_tracker- def set_mission_manager(self, mission_manager: MissionManagerAbstract):- self._mission_manager = mission_manager+ def set_pipeline_manager(self, pipeline_manager: PipelineManagerAbstract):+ self._pipeline_manager = pipeline_managerdef set_activity_manager(self, activity_manager: ActivityManagerProtocol):self._activity_manager = activity_manager@@ -246,15 +246,15 @@ class PipelexHub:raise RuntimeError("PipeRouter is not initialized")return self._pipe_router- def get_mission_tracker(self) -> MissionTrackerProtocol:- if self._mission_tracker is None:- raise RuntimeError("MissionTracker is not initialized")- return self._mission_tracker+ def get_pipeline_tracker(self) -> PipelineTrackerProtocol:+ if self._pipeline_tracker is None:+ raise RuntimeError("PipelineTracker is not initialized")+ return self._pipeline_tracker- def get_required_mission_manager(self) -> MissionManagerAbstract:- if self._mission_manager is None:- raise RuntimeError("MissionManager is not initialized")- return self._mission_manager+ def get_required_pipeline_manager(self) -> PipelineManagerAbstract:+ if self._pipeline_manager is None:+ raise RuntimeError("PipelineManager is not initialized")+ return self._pipeline_managerdef get_activity_manager(self) -> ActivityManagerProtocol:if self._activity_manager is None:@@ -410,17 +410,17 @@ def get_pipe_router() -> PipeRouterProtocol:return get_pipelex_hub().get_required_pipe_router()-def get_mission_tracker() -> MissionTrackerProtocol:- return get_pipelex_hub().get_mission_tracker()+def get_pipeline_tracker() -> PipelineTrackerProtocol:+ return get_pipelex_hub().get_pipeline_tracker()-def get_mission_manager() -> MissionManagerAbstract:- return get_pipelex_hub().get_required_mission_manager()+def get_pipeline_manager() -> PipelineManagerAbstract:+ return get_pipelex_hub().get_required_pipeline_manager()def get_activity_manager() -> ActivityManagerProtocol:return get_pipelex_hub().get_activity_manager()-def get_mission(mission_id: str) -> Mission:- return get_mission_manager().get_mission(mission_id=mission_id)+def get_pipeline(pipeline_run_id: str) -> Pipeline:+ return get_pipeline_manager().get_pipeline(pipeline_run_id=pipeline_run_id)diff --git a/pipelex/mission/execute.py b/pipelex/mission/execute.pydeleted file mode 100644index 483f53b9d..000000000--- a/pipelex/mission/execute.py+++ /dev/null@@ -1,46 +0,0 @@-from typing import Optional, Tuple--from pipelex import pretty_print-from pipelex.core.pipe_output import PipeOutput-from pipelex.core.pipe_run_params import PipeOutputMultiplicity-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory-from pipelex.core.working_memory import WorkingMemory-from pipelex.hub import get_mission_manager, get_pipe_router, get_report_delegate, get_required_pipe-from pipelex.mission.job_metadata import JobMetadata-from pipelex.pipe_works.pipe_job_factory import PipeJobFactory---async def execute_mission(- pipe_code: str,- working_memory: Optional[WorkingMemory] = None,- output_name: Optional[str] = None,- output_multiplicity: Optional[PipeOutputMultiplicity] = None,- dynamic_output_concept_code: Optional[str] = None,-) -> Tuple[PipeOutput, str]:- mission = get_mission_manager().add_new_mission()- mission_id = mission.mission_id- get_report_delegate().open_registry(mission_id=mission_id)- pipe = get_required_pipe(pipe_code=pipe_code)-- job_metadata = JobMetadata(- mission_id=mission_id,- )-- pipe_run_params = PipeRunParamsFactory.make_run_params(- output_multiplicity=output_multiplicity,- dynamic_output_concept_code=dynamic_output_concept_code,- )-- pretty_print(pipe, title=f"Running pipe '{pipe_code}'")- if working_memory:- working_memory.pretty_print_summary()-- pipe_job = PipeJobFactory.make_pipe_job(- pipe=pipe,- pipe_run_params=pipe_run_params,- job_metadata=job_metadata,- working_memory=working_memory,- output_name=output_name,- )-- return await get_pipe_router().run_pipe_job(pipe_job), mission_iddiff --git a/pipelex/mission/mission.py b/pipelex/mission/mission.pydeleted file mode 100644index 4ff33840e..000000000--- a/pipelex/mission/mission.py+++ /dev/null@@ -1,5 +0,0 @@-from pydantic import BaseModel---class Mission(BaseModel):- mission_id: strdiff --git a/pipelex/mission/mission_factory.py b/pipelex/mission/mission_factory.pydeleted file mode 100644index e293e72d4..000000000--- a/pipelex/mission/mission_factory.py+++ /dev/null@@ -1,11 +0,0 @@-import shortuuid--from pipelex.mission.mission import Mission---class MissionFactory:- @classmethod- def make_mission(cls) -> Mission:- return Mission(- mission_id=shortuuid.uuid(),- )diff --git a/pipelex/mission/mission_manager.py b/pipelex/mission/mission_manager.pydeleted file mode 100644index 4ca834966..000000000--- a/pipelex/mission/mission_manager.py+++ /dev/null@@ -1,44 +0,0 @@-from typing import Dict, Optional--from pydantic import Field, RootModel-from typing_extensions import override--from pipelex.exceptions import MissionManagerNotFoundError-from pipelex.mission.mission import Mission-from pipelex.mission.mission_factory import MissionFactory-from pipelex.mission.mission_manager_abstract import MissionManagerAbstract--MissionManagerRoot = Dict[str, Mission]---class MissionManager(MissionManagerAbstract, RootModel[MissionManagerRoot]):- root: MissionManagerRoot = Field(default_factory=dict)-- @override- def setup(self):- pass-- @override- def teardown(self):- self.root.clear()-- @override- def get_optional_mission(self, mission_id: str) -> Optional[Mission]:- return self.root.get(mission_id)-- @override- def get_mission(self, mission_id: str) -> Mission:- mission = self.get_optional_mission(mission_id=mission_id)- if mission is None:- raise MissionManagerNotFoundError(f"Mission {mission_id} not found")- return mission-- def _set_mission(self, mission_id: str, mission: Mission) -> Mission:- self.root[mission_id] = mission- return mission-- @override- def add_new_mission(self) -> Mission:- mission = MissionFactory.make_mission()- self._set_mission(mission_id=mission.mission_id, mission=mission)- return missiondiff --git a/pipelex/pipe_controllers/pipe_batch.py b/pipelex/pipe_controllers/pipe_batch.pyindex 4c0074632..e59ed65f7 100644--- a/pipelex/pipe_controllers/pipe_batch.py+++ b/pipelex/pipe_controllers/pipe_batch.py@@ -14,8 +14,8 @@ from pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeExecutionErrorfrom pipelex.hub import get_pipe_router-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_controllers.pipe_controller import PipeController+from pipelex.pipeline.job_metadata import JobMetadataclass PipeBatch(PipeController):@@ -122,7 +122,7 @@ class PipeBatch(PipeController):# for branch_index, (required_stuff_list, item_input_stuff, item_output_stuff) in enumerate(# zip(required_stuff_lists, item_stuffs, output_stuffs)# ):- # get_mission_tracker().add_batch_step(+ # get_pipeline_tracker().add_batch_step(# from_stuff=input_stuff,# to_stuff=item_input_stuff,# to_branch_index=branch_index,@@ -130,7 +130,7 @@ class PipeBatch(PipeController):# comment="PipeBatch.run_pipe() in zip",# )# for required_stuff in required_stuff_list:- # get_mission_tracker().add_pipe_step(+ # get_pipeline_tracker().add_pipe_step(# from_stuff=required_stuff,# to_stuff=item_output_stuff,# pipe_code=self.branch_pipe_code,@@ -142,7 +142,7 @@ class PipeBatch(PipeController):# for branch_index, branch_output_stuff in enumerate(output_stuffs):# branch_output_item_code = branch_output_item_codes[branch_index]- # get_mission_tracker().add_aggregate_step(+ # get_pipeline_tracker().add_aggregate_step(# from_stuff=branch_output_stuff,# to_stuff=output_stuff,# pipe_layer=pipe_run_params.pipe_layers,diff --git a/pipelex/pipe_controllers/pipe_condition.py b/pipelex/pipe_controllers/pipe_condition.pyindex 61d1fad87..6bef2edfe 100644--- a/pipelex/pipe_controllers/pipe_condition.py+++ b/pipelex/pipe_controllers/pipe_condition.py@@ -9,11 +9,11 @@ from pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeConditionError, PipeDefinitionError, PipeExecutionError, PipeInputError, WorkingMemoryStuffNotFoundError-from pipelex.hub import get_mission_tracker, get_pipe_router-from pipelex.mission.job_metadata import JobCategory, JobMetadata+from pipelex.hub import get_pipe_router, get_pipeline_trackerfrom pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetailsfrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_operators.pipe_jinja2 import PipeJinja2, PipeJinja2Output+from pipelex.pipeline.job_metadata import JobCategory, JobMetadatafrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list@@ -130,7 +130,7 @@ class PipeCondition(PipeController):raise PipeInputError(f"Some required stuff(s) not found - {error_details}") from excfor required_stuff in required_stuffs:- get_mission_tracker().add_condition_step(+ get_pipeline_tracker().add_condition_step(from_stuff=required_stuff,to_condition=condition_details,condition_expression=self.expression or self.applied_expression_jinja2,@@ -146,7 +146,7 @@ class PipeCondition(PipeController):pipe_run_params=pipe_run_params,output_name=output_name,)- get_mission_tracker().add_choice_step(+ get_pipeline_tracker().add_choice_step(from_condition=condition_details,to_stuff=pipe_output.main_stuff,pipe_layer=pipe_run_params.pipe_layers,diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.pyindex d56456f13..ae2445bdf 100644--- a/pipelex/pipe_controllers/pipe_controller.py+++ b/pipelex/pipe_controllers/pipe_controller.py@@ -7,7 +7,7 @@ from pipelex.core.pipe_abstract import PipeAbstractfrom pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemory-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadataclass PipeController(PipeAbstract):diff --git a/pipelex/pipe_controllers/pipe_parallel.py b/pipelex/pipe_controllers/pipe_parallel.pyindex b8c3d49bb..58d8025d2 100644--- a/pipelex/pipe_controllers/pipe_parallel.py+++ b/pipelex/pipe_controllers/pipe_parallel.py@@ -11,10 +11,10 @@ from pipelex.core.stuff_content import StuffContentfrom pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeDefinitionError-from pipelex.hub import get_mission_tracker-from pipelex.mission.job_metadata import JobMetadata+from pipelex.hub import get_pipeline_trackerfrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sub_pipe import SubPipe+from pipelex.pipeline.job_metadata import JobMetadataclass PipeParallel(PipeController):@@ -93,7 +93,7 @@ class PipeParallel(PipeController):name=output_name,)for stuff in output_stuffs.values():- get_mission_tracker().add_aggregate_step(+ get_pipeline_tracker().add_aggregate_step(from_stuff=stuff,to_stuff=combined_output_stuff,pipe_layer=pipe_run_params.pipe_layers,diff --git a/pipelex/pipe_controllers/pipe_sequence.py b/pipelex/pipe_controllers/pipe_sequence.pyindex 96c0ca86e..30e1f8c59 100644--- a/pipelex/pipe_controllers/pipe_sequence.py+++ b/pipelex/pipe_controllers/pipe_sequence.py@@ -7,9 +7,9 @@ from pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeRunParamsError-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_controllers.pipe_controller import PipeControllerfrom pipelex.pipe_controllers.sub_pipe import SubPipe+from pipelex.pipeline.job_metadata import JobMetadataclass PipeSequence(PipeController):diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.pyindex 73f17e6f5..e8bf8c93a 100644--- a/pipelex/pipe_controllers/sub_pipe.py+++ b/pipelex/pipe_controllers/sub_pipe.py@@ -7,10 +7,10 @@ from pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunParamsfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeInputError, WorkingMemoryStuffNotFoundError-from pipelex.hub import get_mission_tracker, get_pipe_router, get_required_pipe-from pipelex.mission.job_metadata import JobMetadata+from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipefrom pipelex.pipe_controllers.pipe_batch import PipeBatchfrom pipelex.pipe_controllers.pipe_condition import PipeCondition+from pipelex.pipeline.job_metadata import JobMetadata# TODO: decide if SubPipe should be a PipeAbstract (it's probably the case)@@ -79,7 +79,7 @@ class SubPipe(BaseModel):)new_output_stuff = pipe_output.main_stufffor stuff in required_stuffs:- get_mission_tracker().add_pipe_step(+ get_pipeline_tracker().add_pipe_step(from_stuff=stuff,to_stuff=new_output_stuff,pipe_code=self.pipe_code,diff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.pyindex 7b263e5c5..b805cc98a 100644--- a/pipelex/pipe_operators/pipe_func.py+++ b/pipelex/pipe_operators/pipe_func.py@@ -8,8 +8,8 @@ from pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.stuff_content import ListContent, StuffContent, TextContentfrom pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import WorkingMemory-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.func_registry import func_registrydiff --git a/pipelex/pipe_operators/pipe_img_gen.py b/pipelex/pipe_operators/pipe_img_gen.pyindex 02907e9de..caabb0570 100644--- a/pipelex/pipe_operators/pipe_img_gen.py+++ b/pipelex/pipe_operators/pipe_img_gen.py@@ -16,8 +16,8 @@ from pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeDefinitionError, PipeInputError, PipeRunParamsError, WorkingMemoryStuffNotFoundErrorfrom pipelex.hub import get_content_generator-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipeline.job_metadata import JobMetadata# TODO: refacto this as part of the PipeImgGen blueprint/paramsIMGG_PROMPT_NAME = "imgg_prompt"diff --git a/pipelex/pipe_operators/pipe_jinja2.py b/pipelex/pipe_operators/pipe_jinja2.pyindex 1807dfc89..11d6ea9ba 100644--- a/pipelex/pipe_operators/pipe_jinja2.py+++ b/pipelex/pipe_operators/pipe_jinja2.py@@ -14,8 +14,8 @@ from pipelex.core.stuff_content import TextContentfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeDefinitionError, PipeRunParamsErrorfrom pipelex.hub import get_content_generator, get_template, get_template_provider-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.templating.jinja2_errors import Jinja2TemplateErrorfrom pipelex.tools.templating.jinja2_parsing import check_jinja2_parsingfrom pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variablesdiff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.pyindex 3c14954ba..951529fd3 100644--- a/pipelex/pipe_operators/pipe_llm.py+++ b/pipelex/pipe_operators/pipe_llm.py@@ -35,11 +35,11 @@ from pipelex.hub import (get_required_pipe,get_template,)-from pipelex.mission.job_metadata import JobCategory, JobMetadatafrom pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Factoryfrom pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutputfrom pipelex.pipe_operators.pipe_operator import PipeOperatorfrom pipelex.pipe_operators.piped_llm_prompt_factory import PipedLLMPromptFactory+from pipelex.pipeline.job_metadata import JobCategory, JobMetadatafrom pipelex.types import StrEnumdiff --git a/pipelex/pipe_operators/pipe_llm_prompt.py b/pipelex/pipe_operators/pipe_llm_prompt.pyindex 750d4e525..87e6d0b4d 100644--- a/pipelex/pipe_operators/pipe_llm_prompt.py+++ b/pipelex/pipe_operators/pipe_llm_prompt.py@@ -24,9 +24,9 @@ from pipelex.exceptions import (WorkingMemoryTypeError,)from pipelex.hub import get_template-from pipelex.mission.job_metadata import JobCategory, JobMetadatafrom pipelex.pipe_operators.pipe_jinja2 import PipeJinja2, PipeJinja2Outputfrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipeline.job_metadata import JobCategory, JobMetadatafrom pipelex.tools.templating.templating_models import PromptingStylefrom pipelex.tools.typing.type_inspector import get_type_structurefrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list, has_more_than_one_among_attributes_from_listdiff --git a/pipelex/pipe_operators/pipe_ocr.py b/pipelex/pipe_operators/pipe_ocr.pyindex 4ed74042b..721b68f8f 100644--- a/pipelex/pipe_operators/pipe_ocr.py+++ b/pipelex/pipe_operators/pipe_ocr.py@@ -15,8 +15,8 @@ from pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.exceptions import PipeDefinitionErrorfrom pipelex.hub import get_content_generator-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_operators.pipe_operator import PipeOperator+from pipelex.pipeline.job_metadata import JobMetadatafrom pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_rendererfrom pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_listdiff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.pyindex 7b7b0d63e..68ac38318 100644--- a/pipelex/pipe_operators/pipe_operator.py+++ b/pipelex/pipe_operators/pipe_operator.py@@ -8,8 +8,8 @@ from pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.hub import get_activity_manager-from pipelex.mission.activity.activity_models import ActivityReport-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.activity.activity_models import ActivityReport+from pipelex.pipeline.job_metadata import JobMetadataclass PipeOperator(PipeAbstract):diff --git a/pipelex/pipe_operators/piped_llm_prompt_factory.py b/pipelex/pipe_operators/piped_llm_prompt_factory.pyindex 4b43bcfb4..542b9d435 100644--- a/pipelex/pipe_operators/piped_llm_prompt_factory.py+++ b/pipelex/pipe_operators/piped_llm_prompt_factory.py@@ -8,8 +8,8 @@ from pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputsfrom pipelex.core.pipe_output import PipeOutputfrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.working_memory_factory import WorkingMemoryFactory-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutput+from pipelex.pipeline.job_metadata import JobMetadataclass PipedLLMPromptFactory(LLMPromptFactoryAbstract):diff --git a/pipelex/pipe_works/pipe_job.py b/pipelex/pipe_works/pipe_job.pyindex 90bdf326b..6454c9577 100644--- a/pipelex/pipe_works/pipe_job.py+++ b/pipelex/pipe_works/pipe_job.py@@ -5,7 +5,7 @@ from pydantic import BaseModel, Fieldfrom pipelex.core.pipe_abstract import PipeAbstractfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemory-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadataclass PipeJob(BaseModel):diff --git a/pipelex/pipe_works/pipe_job_factory.py b/pipelex/pipe_works/pipe_job_factory.pyindex 30e75bf5e..c9e2f8801 100644--- a/pipelex/pipe_works/pipe_job_factory.py+++ b/pipelex/pipe_works/pipe_job_factory.py@@ -4,8 +4,8 @@ from pipelex.core.pipe_abstract import PipeAbstractfrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.working_memory import WorkingMemory-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_works.pipe_job import PipeJob+from pipelex.pipeline.job_metadata import JobMetadataclass PipeJobFactory:diff --git a/pipelex/pipe_works/pipe_router.py b/pipelex/pipe_works/pipe_router.pyindex 233a50326..9217367f0 100644--- a/pipelex/pipe_works/pipe_router.py+++ b/pipelex/pipe_works/pipe_router.py@@ -7,9 +7,9 @@ from pipelex.core.pipe_output import PipeOutputTypefrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.hub import get_required_pipe-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_works.pipe_job_factory import PipeJobFactoryfrom pipelex.pipe_works.pipe_router_protocol import PipeJob, PipeRouterProtocol+from pipelex.pipeline.job_metadata import JobMetadataclass PipeRouter(PipeRouterProtocol):@@ -19,7 +19,7 @@ class PipeRouter(PipeRouterProtocol):pipe_job: PipeJob,wfid: Optional[str] = None,) -> PipeOutputType: # pyright: ignore[reportInvalidTypeVarUse]- log.debug(f"PipeRouter run_pipe_job: pipe_code={pipe_job.pipe.code}")+ log.debug(f"Start run_pipe_job: pipe_code={pipe_job.pipe.code}")working_memory = pipe_job.working_memorypipe = pipe_job.pipe@@ -31,6 +31,7 @@ class PipeRouter(PipeRouterProtocol):output_name=pipe_job.output_name,pipe_run_params=pipe_job.pipe_run_params,)+ log.debug(f"Completed run_pipe_job: pipe_code={pipe_job.pipe.code}")return cast(PipeOutputType, pipe_output)@override@@ -43,7 +44,7 @@ class PipeRouter(PipeRouterProtocol):output_name: Optional[str] = None,wfid: Optional[str] = None,) -> PipeOutputType: # pyright: ignore[reportInvalidTypeVarUse]- log.debug(f"PipeRouter run_pipe_code: pipe_code={pipe_code}")+ log.debug(f"Start run_pipe_code: pipe_code={pipe_code}")pipe = get_required_pipe(pipe_code)pipe_job = PipeJobFactory.make_pipe_job(pipe=pipe,@@ -56,4 +57,5 @@ class PipeRouter(PipeRouterProtocol):pipe_job=pipe_job,wfid=wfid,)+ log.debug(f"Completed run_pipe_code: pipe_code={pipe_code}")return pipe_outputdiff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.pyindex df9b01a4e..bc9c4ea90 100644--- a/pipelex/pipe_works/pipe_router_protocol.py+++ b/pipelex/pipe_works/pipe_router_protocol.py@@ -3,8 +3,8 @@ from typing import Optional, Protocolfrom pipelex.core.pipe_output import PipeOutputTypefrom pipelex.core.pipe_run_params import PipeRunParamsfrom pipelex.core.working_memory import WorkingMemory-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_works.pipe_job import PipeJob+from pipelex.pipeline.job_metadata import JobMetadataclass PipeRouterProtocol(Protocol):diff --git a/pipelex/pipelex.py b/pipelex/pipelex.pyindex 066918893..5a62d1c23 100644--- a/pipelex/pipelex.py+++ b/pipelex/pipelex.py@@ -20,13 +20,13 @@ from pipelex.core.registry_models import PipelexRegistryModelsfrom pipelex.exceptions import PipelexConfigError, PipelexSetupErrorfrom pipelex.hub import PipelexHub, set_pipelex_hubfrom pipelex.libraries.library_manager import LibraryManager-from pipelex.mission.activity.activity_manager import ActivityManager-from pipelex.mission.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol-from pipelex.mission.mission_manager import MissionManager-from pipelex.mission.track.mission_tracker import MissionTracker-from pipelex.mission.track.mission_tracker_protocol import MissionTrackerNoOp, MissionTrackerProtocolfrom pipelex.pipe_works.pipe_router import PipeRouterfrom pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol+from pipelex.pipeline.activity.activity_manager import ActivityManager+from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol+from pipelex.pipeline.pipeline_manager import PipelineManager+from pipelex.pipeline.track.pipeline_tracker import PipelineTracker+from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerNoOp, PipelineTrackerProtocolfrom pipelex.test_extras.registry_test_models import PipelexTestModelsfrom pipelex.tools.config.models import ConfigRootfrom pipelex.tools.func_registry import func_registry@@ -53,8 +53,8 @@ class Pipelex:llm_model_provider: Optional[LLMModelLibrary] = None,plugin_manager: Optional[PluginManager] = None,inference_manager: Optional[InferenceManager] = None,- mission_manager: Optional[MissionManager] = None,- mission_tracker: Optional[MissionTracker] = None,+ pipeline_manager: Optional[PipelineManager] = None,+ pipeline_tracker: Optional[PipelineTracker] = None,activity_manager: Optional[ActivityManagerProtocol] = None,) -> Self:if cls._pipelex_instance is not None:@@ -79,8 +79,8 @@ class Pipelex:llm_model_provider: Optional[LLMModelLibrary] = None,plugin_manager: Optional[PluginManager] = None,inference_manager: Optional[InferenceManager] = None,- mission_manager: Optional[MissionManager] = None,- mission_tracker: Optional[MissionTracker] = None,+ pipeline_manager: Optional[PipelineManager] = None,+ pipeline_tracker: Optional[PipelineTracker] = None,activity_manager: Optional[ActivityManagerProtocol] = None,) -> None:self.pipelex_hub = pipelex_hub or PipelexHub()@@ -128,17 +128,17 @@ class Pipelex:self.pipelex_hub.set_concept_provider(concept_provider=self.library_manager.concept_library)self.pipelex_hub.set_pipe_provider(pipe_provider=self.library_manager.pipe_library)- # pipelex mission- self.mission_tracker: MissionTrackerProtocol- if mission_tracker:- self.mission_tracker = mission_tracker- elif get_config().pipelex.feature_config.is_mission_tracking_enabled:- self.mission_tracker = MissionTracker(tracker_config=get_config().pipelex.tracker_config)+ # pipelex pipeline+ self.pipeline_tracker: PipelineTrackerProtocol+ if pipeline_tracker:+ self.pipeline_tracker = pipeline_tracker+ elif get_config().pipelex.feature_config.is_pipeline_tracking_enabled:+ self.pipeline_tracker = PipelineTracker(tracker_config=get_config().pipelex.tracker_config)else:- self.mission_tracker = MissionTrackerNoOp()- self.pipelex_hub.set_mission_tracker(mission_tracker=self.mission_tracker)- self.mission_manager = mission_manager or MissionManager()- self.pipelex_hub.set_mission_manager(mission_manager=self.mission_manager)+ self.pipeline_tracker = PipelineTrackerNoOp()+ self.pipelex_hub.set_pipeline_tracker(pipeline_tracker=self.pipeline_tracker)+ self.pipeline_manager = pipeline_manager or PipelineManager()+ self.pipelex_hub.set_pipeline_manager(pipeline_manager=self.pipeline_manager)self.activity_manager: ActivityManagerProtocolif activity_manager:@@ -176,9 +176,9 @@ class Pipelex:self.pipelex_hub.set_pipe_router(pipe_router or PipeRouter())- # mission- self.mission_tracker.setup()- self.mission_manager.setup()+ # pipeline+ self.pipeline_tracker.setup()+ self.pipeline_manager.setup()log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup done for {get_config().project_name}")@@ -206,8 +206,8 @@ class Pipelex:def teardown(self):# pipelex- self.mission_manager.teardown()- self.mission_tracker.teardown()+ self.pipeline_manager.teardown()+ self.pipeline_tracker.teardown()self.library_manager.teardown()self.template_provider.teardown()self.activity_manager.teardown()diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 1004cb6b8..a67c11ad8 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -5,7 +5,7 @@ extra_env_files = [][pipelex.feature_config]# WIP/Experimental feature flags-is_mission_tracking_enabled = false+is_pipeline_tracking_enabled = falseis_activity_tracking_enabled = false####################################################################################################diff --git a/pipelex/mission/activity/__init__.py b/pipelex/pipeline/activity/__init__.pysimilarity index 100%rename from pipelex/mission/activity/__init__.pyrename to pipelex/pipeline/activity/__init__.pydiff --git a/pipelex/mission/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.pysimilarity index 98%rename from pipelex/mission/activity/activity_handler.pyrename to pipelex/pipeline/activity/activity_handler.pyindex 4e072775a..904efc6bb 100644--- a/pipelex/mission/activity/activity_handler.py+++ b/pipelex/pipeline/activity/activity_handler.py@@ -13,7 +13,7 @@ from pipelex.core.stuff_content import (StructuredContent,TextContent,)-from pipelex.mission.activity.activity_models import ActivityReport+from pipelex.pipeline.activity.activity_models import ActivityReportfrom pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpxfrom pipelex.tools.misc.file_utils import ensure_path, save_text_to_pathfrom pipelex.tools.misc.json_utils import save_as_json_to_pathdiff --git a/pipelex/mission/activity/activity_manager.py b/pipelex/pipeline/activity/activity_manager.pysimilarity index 87%rename from pipelex/mission/activity/activity_manager.pyrename to pipelex/pipeline/activity/activity_manager.pyindex 4f6f71ef9..b2ba80029 100644--- a/pipelex/mission/activity/activity_manager.py+++ b/pipelex/pipeline/activity/activity_manager.py@@ -4,8 +4,8 @@ from pydantic import Fieldfrom typing_extensions import overridefrom pipelex import log-from pipelex.mission.activity.activity_manager_protocol import ActivityManagerProtocol-from pipelex.mission.activity.activity_models import ActivityCallback, ActivityReport+from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol+from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReportfrom pipelex.tools.exceptions import RootExceptiondiff --git a/pipelex/mission/activity/activity_manager_protocol.py b/pipelex/pipeline/activity/activity_manager_protocol.pysimilarity index 90%rename from pipelex/mission/activity/activity_manager_protocol.pyrename to pipelex/pipeline/activity/activity_manager_protocol.pyindex 81783b55e..a950b9281 100644--- a/pipelex/mission/activity/activity_manager_protocol.py+++ b/pipelex/pipeline/activity/activity_manager_protocol.py@@ -2,7 +2,7 @@ from typing import Protocolfrom typing_extensions import override-from pipelex.mission.activity.activity_models import ActivityCallback, ActivityReport+from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReportclass ActivityManagerProtocol(Protocol):diff --git a/pipelex/mission/activity/activity_models.py b/pipelex/pipeline/activity/activity_models.pysimilarity index 78%rename from pipelex/mission/activity/activity_models.pyrename to pipelex/pipeline/activity/activity_models.pyindex e99c93bc5..afc8a37b9 100644--- a/pipelex/mission/activity/activity_models.py+++ b/pipelex/pipeline/activity/activity_models.py@@ -2,7 +2,7 @@ from typing import Any, Callablefrom pydantic import BaseModel-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadataclass ActivityReport(BaseModel):diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.pynew file mode 100644index 000000000..74a7ace42--- /dev/null+++ b/pipelex/pipeline/execute.py@@ -0,0 +1,70 @@+from typing import Optional, Tuple++from pipelex import pretty_print+from pipelex.core.pipe_output import PipeOutput+from pipelex.core.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.working_memory import WorkingMemory+from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe+from pipelex.pipe_works.pipe_job_factory import PipeJobFactory+from pipelex.pipeline.job_metadata import JobMetadata+++async def execute_pipeline(+ pipe_code: str,+ working_memory: Optional[WorkingMemory] = None,+ output_name: Optional[str] = None,+ output_multiplicity: Optional[PipeOutputMultiplicity] = None,+ dynamic_output_concept_code: Optional[str] = None,+) -> Tuple[PipeOutput, str]:+ """Execute a pipeline and wait for its completion.++ This function executes a pipe and returns its output along with the pipeline run ID.+ Unlike *start_pipeline*, this function waits for the pipe execution to complete+ before returning, and it returns the output in addition to the pipeline run ID.++ Parameters+ ----------+ pipe_code:+ The code of the pipe to execute.+ working_memory:+ Optional ``WorkingMemory`` instance passed to the pipe.+ output_name:+ Name of the output slot to write to.+ output_multiplicity:+ Output multiplicity.+ dynamic_output_concept_code:+ Override the dynamic output concept code.++ Returns+ -------+ Tuple[PipeOutput, str]+ A tuple containing the pipe output and the pipeline run ID.+ """+ pipeline = get_pipeline_manager().add_new_pipeline()+ pipeline_run_id = pipeline.pipeline_run_id+ get_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)+ pipe = get_required_pipe(pipe_code=pipe_code)++ job_metadata = JobMetadata(+ pipeline_run_id=pipeline_run_id,+ )++ pipe_run_params = PipeRunParamsFactory.make_run_params(+ output_multiplicity=output_multiplicity,+ dynamic_output_concept_code=dynamic_output_concept_code,+ )++ pretty_print(pipe, title=f"Running pipe '{pipe_code}'")+ if working_memory:+ working_memory.pretty_print_summary()++ pipe_job = PipeJobFactory.make_pipe_job(+ pipe=pipe,+ pipe_run_params=pipe_run_params,+ job_metadata=job_metadata,+ working_memory=working_memory,+ output_name=output_name,+ )++ return await get_pipe_router().run_pipe_job(pipe_job), pipeline_run_iddiff --git a/pipelex/mission/job_metadata.py b/pipelex/pipeline/job_metadata.pysimilarity index 94%rename from pipelex/mission/job_metadata.pyrename to pipelex/pipeline/job_metadata.pyindex ee74a0d98..104d60a2b 100644--- a/pipelex/mission/job_metadata.py+++ b/pipelex/pipeline/job_metadata.py@@ -3,7 +3,7 @@ from typing import List, Optionalfrom pydantic import BaseModel, ConfigDict, Field-from pipelex.mission.mission_models import SpecialMissionId+from pipelex.pipeline.pipeline_models import SpecialPipelineIdfrom pipelex.types import StrEnum@@ -26,7 +26,7 @@ class UnitJobId(StrEnum):class JobMetadata(BaseModel):model_config = ConfigDict(arbitrary_types_allowed=True)- mission_id: str = Field(default=SpecialMissionId.UNTITLED)+ pipeline_run_id: str = Field(default=SpecialPipelineId.UNTITLED)top_job_id: Optional[str] = Nonepipe_job_ids: Optional[List[str]] = Nonecontent_generation_job_id: Optional[str] = Nonediff --git a/pipelex/pipeline/pipeline.py b/pipelex/pipeline/pipeline.pynew file mode 100644index 000000000..c16495476--- /dev/null+++ b/pipelex/pipeline/pipeline.py@@ -0,0 +1,5 @@+from pydantic import BaseModel+++class Pipeline(BaseModel):+ pipeline_run_id: strdiff --git a/pipelex/pipeline/pipeline_factory.py b/pipelex/pipeline/pipeline_factory.pynew file mode 100644index 000000000..e56931907--- /dev/null+++ b/pipelex/pipeline/pipeline_factory.py@@ -0,0 +1,11 @@+import shortuuid++from pipelex.pipeline.pipeline import Pipeline+++class PipelineFactory:+ @classmethod+ def make_pipeline(cls) -> Pipeline:+ return Pipeline(+ pipeline_run_id=shortuuid.uuid(),+ )diff --git a/pipelex/pipeline/pipeline_manager.py b/pipelex/pipeline/pipeline_manager.pynew file mode 100644index 000000000..99600a1c0--- /dev/null+++ b/pipelex/pipeline/pipeline_manager.py@@ -0,0 +1,44 @@+from typing import Dict, Optional++from pydantic import Field, RootModel+from typing_extensions import override++from pipelex.exceptions import PipelineManagerNotFoundError+from pipelex.pipeline.pipeline import Pipeline+from pipelex.pipeline.pipeline_factory import PipelineFactory+from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract++PipelineManagerRoot = Dict[str, Pipeline]+++class PipelineManager(PipelineManagerAbstract, RootModel[PipelineManagerRoot]):+ root: PipelineManagerRoot = Field(default_factory=dict)++ @override+ def setup(self):+ pass++ @override+ def teardown(self):+ self.root.clear()++ @override+ def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:+ return self.root.get(pipeline_run_id)++ @override+ def get_pipeline(self, pipeline_run_id: str) -> Pipeline:+ pipeline = self.get_optional_pipeline(pipeline_run_id=pipeline_run_id)+ if pipeline is None:+ raise PipelineManagerNotFoundError(f"Pipeline {pipeline_run_id} not found")+ return pipeline++ def _set_pipeline(self, pipeline_run_id: str, pipeline: Pipeline) -> Pipeline:+ self.root[pipeline_run_id] = pipeline+ return pipeline++ @override+ def add_new_pipeline(self) -> Pipeline:+ pipeline = PipelineFactory.make_pipeline()+ self._set_pipeline(pipeline_run_id=pipeline.pipeline_run_id, pipeline=pipeline)+ return pipelinediff --git a/pipelex/mission/mission_manager_abstract.py b/pipelex/pipeline/pipeline_manager_abstract.pysimilarity index 52%rename from pipelex/mission/mission_manager_abstract.pyrename to pipelex/pipeline/pipeline_manager_abstract.pyindex dfc69684d..1c7e13878 100644--- a/pipelex/mission/mission_manager_abstract.py+++ b/pipelex/pipeline/pipeline_manager_abstract.py@@ -1,10 +1,10 @@from abc import ABC, abstractmethodfrom typing import Optional-from pipelex.mission.mission import Mission+from pipelex.pipeline.pipeline import Pipeline-class MissionManagerAbstract(ABC):+class PipelineManagerAbstract(ABC):@abstractmethoddef setup(self) -> None:pass@@ -14,13 +14,13 @@ class MissionManagerAbstract(ABC):pass@abstractmethod- def get_optional_mission(self, mission_id: str) -> Optional[Mission]:+ def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:pass@abstractmethod- def get_mission(self, mission_id: str) -> Mission:+ def get_pipeline(self, pipeline_run_id: str) -> Pipeline:pass@abstractmethod- def add_new_mission(self) -> Mission:+ def add_new_pipeline(self) -> Pipeline:passdiff --git a/pipelex/mission/mission_models.py b/pipelex/pipeline/pipeline_models.pysimilarity index 64%rename from pipelex/mission/mission_models.pyrename to pipelex/pipeline/pipeline_models.pyindex 4335f69b2..5fd4308e6 100644--- a/pipelex/mission/mission_models.py+++ b/pipelex/pipeline/pipeline_models.py@@ -1,5 +1,5 @@from pipelex.types import StrEnum-class SpecialMissionId(StrEnum):+class SpecialPipelineId(StrEnum):UNTITLED = "untitled"diff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.pynew file mode 100644index 000000000..022539847--- /dev/null+++ b/pipelex/pipeline/start.py@@ -0,0 +1,77 @@+import asyncio+from typing import Optional, Tuple++from pipelex import pretty_print+from pipelex.core.pipe_output import PipeOutput+from pipelex.core.pipe_run_params import PipeOutputMultiplicity+from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory+from pipelex.core.working_memory import WorkingMemory+from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe+from pipelex.pipe_works.pipe_job_factory import PipeJobFactory+from pipelex.pipeline.job_metadata import JobMetadata+++async def start_pipeline(+ pipe_code: str,+ working_memory: Optional[WorkingMemory] = None,+ output_name: Optional[str] = None,+ output_multiplicity: Optional[PipeOutputMultiplicity] = None,+ dynamic_output_concept_code: Optional[str] = None,+) -> Tuple[str, asyncio.Task[PipeOutput]]:+ """Start a pipeline in the background.++ This function mirrors *execute_pipeline* but returns immediately with the+ ``pipeline_run_id`` instead of waiting for the pipe run to complete. The+ actual execution is scheduled on the current event-loop using+ :pyfunc:`asyncio.create_task`.++ Parameters+ ----------+ pipe_code:+ The code of the pipe to execute.+ working_memory:+ Optional ``WorkingMemory`` instance passed to the pipe.+ output_name:+ Name of the output slot to write to.+ output_multiplicity:+ Output multiplicity.+ dynamic_output_concept_code:+ Override the dynamic output concept code.++ Returns+ -------+ Tuple[str, asyncio.Task[PipeOutput]]+ The ``pipeline_run_id`` of the newly started pipeline and a task that+ can be awaited to get the pipe output.+ """++ pipeline = get_pipeline_manager().add_new_pipeline()+ pipeline_run_id = pipeline.pipeline_run_id+ get_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)+ pipe = get_required_pipe(pipe_code=pipe_code)++ job_metadata = JobMetadata(+ pipeline_run_id=pipeline_run_id,+ )++ pipe_run_params = PipeRunParamsFactory.make_run_params(+ output_multiplicity=output_multiplicity,+ dynamic_output_concept_code=dynamic_output_concept_code,+ )++ pretty_print(pipe, title=f"Starting pipe '{pipe_code}' (background)")+ if working_memory:+ working_memory.pretty_print_summary()++ pipe_job = PipeJobFactory.make_pipe_job(+ pipe=pipe,+ pipe_run_params=pipe_run_params,+ job_metadata=job_metadata,+ working_memory=working_memory,+ output_name=output_name,+ )++ # Launch execution without awaiting the result.+ task: asyncio.Task[PipeOutput] = asyncio.create_task(get_pipe_router().run_pipe_job(pipe_job))++ return pipeline_run_id, taskdiff --git a/pipelex/mission/track/__init__.py b/pipelex/pipeline/track/__init__.pysimilarity index 100%rename from pipelex/mission/track/__init__.pyrename to pipelex/pipeline/track/__init__.pydiff --git a/pipelex/mission/track/flow_chart.py b/pipelex/pipeline/track/flow_chart.pysimilarity index 98%rename from pipelex/mission/track/flow_chart.pyrename to pipelex/pipeline/track/flow_chart.pyindex 856c12dd8..6736fb75e 100644--- a/pipelex/mission/track/flow_chart.py+++ b/pipelex/pipeline/track/flow_chart.py@@ -10,8 +10,8 @@ import yamlfrom pipelex import logfrom pipelex.exceptions import JobHistoryError-from pipelex.mission.track.tracker_config import TrackerConfig-from pipelex.mission.track.tracker_models import (+from pipelex.pipeline.track.tracker_config import TrackerConfig+from pipelex.pipeline.track.tracker_models import (EdgeAttributeKey,EdgeCategory,GraphTree,@@ -31,7 +31,7 @@ def _indent_line(line: str, indent: int) -> str:return f"{' ' * indent}{line}"-class MissionFlowChart:+class PipelineFlowChart:def __init__(self, nx_graph: nx.DiGraph, start_node: str, tracker_config: TrackerConfig):self.nx_graph = nx_graphself._tracker_config = tracker_configdiff --git a/pipelex/mission/track/mission_tracker.py b/pipelex/pipeline/track/pipeline_tracker.pysimilarity index 94%rename from pipelex/mission/track/mission_tracker.pyrename to pipelex/pipeline/track/pipeline_tracker.pyindex 832aa529e..0486e4719 100644--- a/pipelex/mission/track/mission_tracker.py+++ b/pipelex/pipeline/track/pipeline_tracker.py@@ -11,23 +11,23 @@ from pipelex import logfrom pipelex.core.concept import Conceptfrom pipelex.core.stuff import Stufffrom pipelex.exceptions import JobHistoryError-from pipelex.mission.track.flow_chart import MissionFlowChart-from pipelex.mission.track.mission_tracker_protocol import MissionTrackerProtocol-from pipelex.mission.track.tracker_config import TrackerConfig-from pipelex.mission.track.tracker_models import (+from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails+from pipelex.pipeline.track.flow_chart import PipelineFlowChart+from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocol+from pipelex.pipeline.track.tracker_config import TrackerConfig+from pipelex.pipeline.track.tracker_models import (EdgeAttributeKey,EdgeCategory,NodeAttributeKey,NodeCategory,SpecialNodeName,)-from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetailsfrom pipelex.tools.misc.mermaid_utils import print_mermaid_url-# TODO: manage a separate graph for each mission_id+# TODO: manage a separate graph for each pipeline_run_id# TODO: restore disabled tracking functionality in PipeBatch-class MissionTracker(MissionTrackerProtocol):+class PipelineTracker(PipelineTrackerProtocol):def __init__(self, tracker_config: TrackerConfig):self._tracker_config = tracker_configself._is_debug_mode = tracker_config.is_debug_mode@@ -320,11 +320,11 @@ class MissionTracker(MissionTrackerProtocol):def _print_mermaid_flowchart_code_and_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):if not self.nx_graph.nodes:- log.info("No nodes in the mission tracker")+ log.info("No nodes in the pipeline tracker")returnif self.start_node is None:raise JobHistoryError("Start node is not set")- flowchart = MissionFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)+ flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)mermaid_code, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)print(mermaid_code)title_to_print = "Mermaid flowchart URL"@@ -334,11 +334,11 @@ class MissionTracker(MissionTrackerProtocol):def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):if not self.nx_graph.nodes:- log.info("No nodes in the mission tracker")+ log.info("No nodes in the pipeline tracker")returnif self.start_node is None:raise JobHistoryError("Start node is not set")- flowchart = MissionFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)+ flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)_, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)title_to_print = "Mermaid flowchart URL"if title:diff --git a/pipelex/mission/track/mission_tracker_protocol.py b/pipelex/pipeline/track/pipeline_tracker_protocol.pysimilarity index 92%rename from pipelex/mission/track/mission_tracker_protocol.pyrename to pipelex/pipeline/track/pipeline_tracker_protocol.pyindex f71400134..95cb08528 100644--- a/pipelex/mission/track/mission_tracker_protocol.py+++ b/pipelex/pipeline/track/pipeline_tracker_protocol.py@@ -10,7 +10,7 @@ from pipelex.core.stuff import Stufffrom pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails-class MissionTrackerProtocol(Protocol):+class PipelineTrackerProtocol(Protocol):def setup(self): ...def teardown(self): ...@@ -69,9 +69,9 @@ class MissionTrackerProtocol(Protocol):): ...-class MissionTrackerNoOp(MissionTrackerProtocol):- """A no-operation implementation of MissionTrackerProtocol that does nothing.- This is useful when mission tracking needs to be disabled.+class PipelineTrackerNoOp(PipelineTrackerProtocol):+ """A no-operation implementation of PipelineTrackerProtocol that does nothing.+ This is useful when pipeline tracking needs to be disabled."""@overridediff --git a/pipelex/mission/track/tracker_config.py b/pipelex/pipeline/track/tracker_config.pysimilarity index 100%rename from pipelex/mission/track/tracker_config.pyrename to pipelex/pipeline/track/tracker_config.pydiff --git a/pipelex/mission/track/tracker_models.py b/pipelex/pipeline/track/tracker_models.pysimilarity index 100%rename from pipelex/mission/track/tracker_models.pyrename to pipelex/pipeline/track/tracker_models.pydiff --git a/pipelex/run.py b/pipelex/run.pyindex 1c3614e05..2b2924204 100644--- a/pipelex/run.py+++ b/pipelex/run.py@@ -7,8 +7,8 @@ from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunParamsfrom pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.hub import get_pipe_router, get_required_pipe-from pipelex.mission.job_metadata import JobMetadatafrom pipelex.pipe_works.pipe_job_factory import PipeJobFactory+from pipelex.pipeline.job_metadata import JobMetadataasync def run_pipe_code(diff --git a/tests/pipelex/cogt/cogt_asynch/test_content_generator.py b/tests/pipelex/cogt/cogt_asynch/test_content_generator.pyindex c6945d651..37503d0a5 100644--- a/tests/pipelex/cogt/cogt_asynch/test_content_generator.py+++ b/tests/pipelex/cogt/cogt_asynch/test_content_generator.py@@ -15,7 +15,7 @@ from pipelex.cogt.ocr.ocr_input import OcrInputfrom pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParamsfrom pipelex.cogt.ocr.ocr_output import OcrOutputfrom pipelex.hub import get_content_generator, get_llm_deck-from pipelex.mission.job_metadata import JobMetadata+from pipelex.pipeline.job_metadata import JobMetadatafrom tests.pipelex.cogt.test_data import Employeefrom tests.test_data import ImageTestCasesdiff --git a/tests/pipelex/pipelex_asynch/conftest.py b/tests/pipelex/pipelex_asynch/conftest.pyindex 0b78a8552..dc04dd01f 100644--- a/tests/pipelex/pipelex_asynch/conftest.py+++ b/tests/pipelex/pipelex_asynch/conftest.py@@ -6,7 +6,7 @@ from pytest import FixtureRequestfrom pipelex.core.pipe_output import PipeOutputfrom pipelex.hub import get_activity_manager-from pipelex.mission.activity.activity_handler import ActivityHandlerForResultFiles+from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFilesfrom pipelex.tools.misc.file_utils import get_incremental_directory_path, remove_folderfrom pipelex.tools.misc.json_utils import save_as_json_to_pathfrom tests.conftest import TEST_OUTPUTS_DIRdiff --git a/tests/pipelex/pipelex_asynch/test_pipe_batch.py b/tests/pipelex/pipelex_asynch/test_pipe_batch.pyindex 4adac5ddb..10fae5f2f 100644--- a/tests/pipelex/pipelex_asynch/test_pipe_batch.py+++ b/tests/pipelex/pipelex_asynch/test_pipe_batch.py@@ -6,7 +6,7 @@ from pipelex.core.pipe_run_params_factory import PipeRunParamsFactoryfrom pipelex.core.stuff_content import ListContent, TextContentfrom pipelex.core.stuff_factory import StuffFactoryfrom pipelex.core.working_memory_factory import WorkingMemoryFactory-from pipelex.hub import get_mission_tracker, get_pipe_router, get_report_delegate+from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_report_delegate@pytest.mark.llm@@ -45,4 +45,4 @@ class TestPipeBatch:assert pipe_output.working_memory is not Noneassert pipe_output.main_stuff is not None- get_mission_tracker().output_flowchart()+ get_pipeline_tracker().output_flowchart()diff --git a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.pyindex 2d7b4bf8c..2bdcf796d 100644--- a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py+++ b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py@@ -11,9 +11,9 @@ from pipelex.core.stuff import Stufffrom pipelex.core.stuff_factory import StuffBlueprintfrom pipelex.core.working_memory import WorkingMemoryfrom pipelex.core.working_memory_factory import WorkingMemoryFactory-from pipelex.hub import get_mission_tracker, get_pipe_router, get_report_delegate-from pipelex.mission.activity.activity_handler import ActivityHandlerForResultFiles-from pipelex.mission.job_metadata import JobMetadata+from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_report_delegate+from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles+from pipelex.pipeline.job_metadata import JobMetadatafrom tests.pipelex.test_data import PipeTestCases@@ -166,7 +166,7 @@ class TestPipeRouter:pretty_print(pipe_output, title=f"run pipe '{pipe_code}'")get_report_delegate().generate_report()- get_mission_tracker().output_flowchart(is_detailed=True)+ get_pipeline_tracker().output_flowchart(is_detailed=True)@pytest.mark.parametrize("pipe_code, exception, expected_error_message", PipeTestCases.FAILURE_PIPES)async def test_pipe_infinite_loop(