Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Refactor/rename pipeline (#45)

Louis Choquelfddda669946d
diff --git a/Makefile b/Makefile
index 52b79909a..707873169 100644
--- a/Makefile
+++ b/Makefile
@@ -65,10 +65,12 @@ make init - Run `pipelex init libraries` and `pipelex init c
make codex-tests - Run tests for Codex (exit on first failure) (no inference, no codex_disabled)
make gha-tests - Run tests for github actions (exit on first failure) (no inference, no gha_disabled)
make test - Run unit tests (no inference)
+make test-xdist - Run unit tests with xdist (no inference)
+make t - Shorthand -> test-xdist
make test-quiet - Run unit tests without prints (no inference)
make tq - Shorthand -> test-quiet
make test-with-prints - Run tests with prints (no inference)
-make t - Shorthand -> test-with-prints
+make tp - Shorthand -> test-with-prints
make test-inference - Run unit tests only for inference (with prints)
make ti - Shorthand -> test-inference
make test-ocr - Run unit tests only for ocr (with prints)
@@ -90,7 +92,7 @@ export HELP
all help env lock install update build \
format lint pyright mypy \
cleanderived cleanenv cleanlibraries cleanall \
- test test-quiet tq test-with-prints t test-inference ti \
+ test test-xdist t test-quiet tq test-with-prints tp test-inference ti \
test-imgg tg test-ocr to codex-tests gha-tests \
run-all-tests run-manual-trigger-gha-tests run-gha_disabled-tests \
validate v check c cc \
@@ -227,18 +229,30 @@ test: env
$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
+test-xdist: env
+ $(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
+ @echo "• Running unit tests"
+ @if [ -n "$(TEST)" ]; then \
+ $(VENV_PYTEST) -n auto -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ else \
+ $(VENV_PYTEST) -n auto -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ fi
+
+t: test-xdist
+ @echo "> done: t = test-xdist"
+
test-quiet: env
$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
tq: test-quiet
@@ -248,20 +262,20 @@ test-with-prints: env
$(call PRINT_TITLE,"Unit testing with prints and our rich logs")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
-t: test-with-prints
- @echo "> done: t = test-with-prints"
+tp: test-with-prints
+ @echo "> done: tp = test-with-prints"
test-inference: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
ti: test-inference
@@ -270,9 +284,9 @@ ti: test-inference
test-ocr: env
$(call PRINT_TITLE,"Unit testing ocr")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
to: test-ocr
@@ -281,9 +295,9 @@ to: test-ocr
test-imgg: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,-v)); \
+ $(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
tg: test-imgg
diff --git a/pipelex/cogt/content_generation/assignment_models.py b/pipelex/cogt/content_generation/assignment_models.py
index 686fbff66..a02d96bb6 100644
--- a/pipelex/cogt/content_generation/assignment_models.py
+++ b/pipelex/cogt/content_generation/assignment_models.py
@@ -16,7 +16,7 @@ from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstrac
from pipelex.cogt.ocr.ocr_handle import OcrHandle
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategory
from pipelex.tools.templating.templating_models import PromptingStyle
diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.py
index 5d1486cd8..89f3a2398 100644
--- a/pipelex/cogt/content_generation/content_generator.py
+++ b/pipelex/cogt/content_generation/content_generator.py
@@ -30,7 +30,7 @@ from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
from pipelex.cogt.ocr.ocr_output import OcrOutput
from pipelex.config import get_config
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategory
from pipelex.tools.templating.templating_models import PromptingStyle
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.py
index f14f6a902..034badbd7 100644
--- a/pipelex/cogt/content_generation/content_generator_protocol.py
+++ b/pipelex/cogt/content_generation/content_generator_protocol.py
@@ -12,7 +12,7 @@ from pipelex.cogt.ocr.ocr_handle import OcrHandle
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
from pipelex.cogt.ocr.ocr_output import OcrOutput
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.templating.jinja2_environment import Jinja2TemplateCategory
from pipelex.tools.templating.templating_models import PromptingStyle
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
diff --git a/pipelex/cogt/imgg/imgg_job_factory.py b/pipelex/cogt/imgg/imgg_job_factory.py
index f416b1725..91fbb8587 100644
--- a/pipelex/cogt/imgg/imgg_job_factory.py
+++ b/pipelex/cogt/imgg/imgg_job_factory.py
@@ -4,7 +4,7 @@ from pipelex.cogt.imgg.imgg_job import ImggJob
from pipelex.cogt.imgg.imgg_job_components import ImggJobConfig, ImggJobParams, ImggJobReport
from pipelex.cogt.imgg.imgg_prompt import ImggPrompt
from pipelex.config import get_config
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
class ImggJobFactory:
diff --git a/pipelex/cogt/imgg/imgg_worker_abstract.py b/pipelex/cogt/imgg/imgg_worker_abstract.py
index 65dc51c41..384afec00 100644
--- a/pipelex/cogt/imgg/imgg_worker_abstract.py
+++ b/pipelex/cogt/imgg/imgg_worker_abstract.py
@@ -10,7 +10,7 @@ from pipelex.cogt.imgg.imgg_engine import ImggEngine
from pipelex.cogt.imgg.imgg_job import ImggJob
from pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegate
from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbstract
-from pipelex.mission.job_metadata import UnitJobId
+from pipelex.pipeline.job_metadata import UnitJobId
F = TypeVar("F", bound=Callable[..., Awaitable[Any]])
diff --git a/pipelex/cogt/inference/cost_registry.py b/pipelex/cogt/inference/cost_registry.py
index 4e024fa10..c8fc821cc 100644
--- a/pipelex/cogt/inference/cost_registry.py
+++ b/pipelex/cogt/inference/cost_registry.py
@@ -28,13 +28,13 @@ class CostRegistry(RootModel[CostRegistryRoot]):
@classmethod
def generate_report(
cls,
- mission_id: str,
+ pipeline_run_id: str,
llm_tokens_usages: List[LLMTokensUsage],
unit_scale: float,
cost_report_file_path: Optional[str] = None,
):
if not llm_tokens_usages:
- log.warning(f"No report to generate for mission '{mission_id}'")
+ log.warning(f"No report to generate for pipeline '{pipeline_run_id}'")
return
cost_registry = CostRegistry()
for llm_tokens_usage in llm_tokens_usages:
@@ -77,7 +77,7 @@ class CostRegistry(RootModel[CostRegistryRoot]):
console = Console()
title = "Costs by LLM model"
- title += f" for mission '{mission_id}'"
+ title += f" for pipeline '{pipeline_run_id}'"
table = Table(title=title, box=box.ROUNDED)
scale_str: str
diff --git a/pipelex/cogt/inference/inference_job_abstract.py b/pipelex/cogt/inference/inference_job_abstract.py
index 9b99655d8..da5dd2b2f 100644
--- a/pipelex/cogt/inference/inference_job_abstract.py
+++ b/pipelex/cogt/inference/inference_job_abstract.py
@@ -2,7 +2,7 @@ from abc import ABC, abstractmethod
from pydantic import BaseModel
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
class InferenceJobAbstract(ABC, BaseModel):
diff --git a/pipelex/cogt/inference/inference_report_delegate.py b/pipelex/cogt/inference/inference_report_delegate.py
index a69bbfbbd..8d5d9bf09 100644
--- a/pipelex/cogt/inference/inference_report_delegate.py
+++ b/pipelex/cogt/inference/inference_report_delegate.py
@@ -4,10 +4,10 @@ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
class InferenceReportDelegate(Protocol):
- def open_registry(self, mission_id: str): ...
+ def open_registry(self, pipeline_run_id: str): ...
def report_inference_job(self, inference_job: InferenceJobAbstract): ...
- def generate_report(self, mission_id: Optional[str] = None): ...
+ def generate_report(self, pipeline_run_id: Optional[str] = None): ...
- def close_registry(self, mission_id: str): ...
+ def close_registry(self, pipeline_run_id: str): ...
diff --git a/pipelex/cogt/inference/inference_report_manager.py b/pipelex/cogt/inference/inference_report_manager.py
index 8457c568f..02f766cc9 100644
--- a/pipelex/cogt/inference/inference_report_manager.py
+++ b/pipelex/cogt/inference/inference_report_manager.py
@@ -11,7 +11,7 @@ from pipelex.cogt.inference.inference_job_abstract import InferenceJobAbstract
from pipelex.cogt.inference.inference_report_delegate import InferenceReportDelegate
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_report import LLMTokenCostReport, LLMTokensUsage
-from pipelex.mission.mission_models import SpecialMissionId
+from pipelex.pipeline.pipeline_models import SpecialPipelineId
from pipelex.tools.misc.file_utils import ensure_path, get_incremental_file_path
LLMUsageRegistryRoot = List[LLMTokensUsage]
@@ -38,7 +38,7 @@ class InferenceReportManager(InferenceReportDelegate):
def setup(self):
self._usage_registries.clear()
- self._usage_registries[SpecialMissionId.UNTITLED] = UsageRegistry()
+ self._usage_registries[SpecialPipelineId.UNTITLED] = UsageRegistry()
def teardown(self):
self._usage_registries.clear()
@@ -47,10 +47,10 @@ class InferenceReportManager(InferenceReportDelegate):
# Private methods
############################################################
- def _get_registry(self, mission_id: str) -> UsageRegistry:
- if mission_id not in self._usage_registries:
- raise InferenceReportManagerError(f"Registry for mission '{mission_id}' does not exist")
- return self._usage_registries[mission_id]
+ def _get_registry(self, pipeline_run_id: str) -> UsageRegistry:
+ if pipeline_run_id not in self._usage_registries:
+ raise InferenceReportManagerError(f"Registry for pipeline '{pipeline_run_id}' does not exist")
+ return self._usage_registries[pipeline_run_id]
def _report_llm_job(self, llm_job: LLMJob):
llm_tokens_usage = llm_job.job_report.llm_tokens_usage
@@ -64,8 +64,8 @@ class InferenceReportManager(InferenceReportDelegate):
if self._report_config.is_log_costs_to_console:
llm_token_cost_report = CostRegistry.complete_cost_report(llm_tokens_usage=llm_tokens_usage)
- mission_id = llm_job.job_metadata.mission_id
- self._get_registry(mission_id).add_tokens_usage(llm_tokens_usage)
+ pipeline_run_id = llm_job.job_metadata.pipeline_run_id
+ self._get_registry(pipeline_run_id).add_tokens_usage(llm_tokens_usage)
if self._report_config.is_log_costs_to_console:
log.verbose(llm_token_cost_report, title="Token Cost report")
@@ -75,10 +75,10 @@ class InferenceReportManager(InferenceReportDelegate):
############################################################
@override
- def open_registry(self, mission_id: str):
- if mission_id in self._usage_registries:
- raise InferenceReportManagerError(f"Registry for mission '{mission_id}' already exists")
- self._usage_registries[mission_id] = UsageRegistry()
+ def open_registry(self, pipeline_run_id: str):
+ if pipeline_run_id in self._usage_registries:
+ raise InferenceReportManagerError(f"Registry for pipeline '{pipeline_run_id}' already exists")
+ self._usage_registries[pipeline_run_id] = UsageRegistry()
@override
def report_inference_job(self, inference_job: InferenceJobAbstract):
@@ -91,8 +91,8 @@ class InferenceReportManager(InferenceReportDelegate):
self._report_llm_job(llm_job=llm_job)
@override
- def generate_report(self, mission_id: Optional[str] = None):
- mission_id = mission_id or SpecialMissionId.UNTITLED
+ def generate_report(self, pipeline_run_id: Optional[str] = None):
+ pipeline_run_id = pipeline_run_id or SpecialPipelineId.UNTITLED
cost_report_file_path: Optional[str] = None
if self._report_config.is_generate_cost_report_file_enabled:
ensure_path(self._report_config.cost_report_dir_path)
@@ -102,14 +102,14 @@ class InferenceReportManager(InferenceReportDelegate):
extension=self._report_config.cost_report_extension,
)
- registry = self._get_registry(mission_id)
+ registry = self._get_registry(pipeline_run_id)
CostRegistry.generate_report(
- mission_id=mission_id,
+ pipeline_run_id=pipeline_run_id,
llm_tokens_usages=registry.get_current_tokens_usage(),
unit_scale=self._report_config.cost_report_unit_scale,
cost_report_file_path=cost_report_file_path,
)
@override
- def close_registry(self, mission_id: str):
- self._usage_registries.pop(mission_id)
+ def close_registry(self, pipeline_run_id: str):
+ self._usage_registries.pop(pipeline_run_id)
diff --git a/pipelex/cogt/llm/llm_job_factory.py b/pipelex/cogt/llm/llm_job_factory.py
index e6c9810ff..f96a99865 100644
--- a/pipelex/cogt/llm/llm_job_factory.py
+++ b/pipelex/cogt/llm/llm_job_factory.py
@@ -8,7 +8,7 @@ from pipelex.cogt.llm.llm_prompt import LLMPrompt
from pipelex.cogt.llm.llm_prompt_factory_abstract import LLMPromptFactoryAbstract
from pipelex.cogt.llm.llm_prompt_template import LLMPromptTemplate
from pipelex.config import get_config
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
class LLMJobFactory:
diff --git a/pipelex/cogt/llm/llm_report.py b/pipelex/cogt/llm/llm_report.py
index f15469888..b373086cb 100644
--- a/pipelex/cogt/llm/llm_report.py
+++ b/pipelex/cogt/llm/llm_report.py
@@ -7,7 +7,7 @@ from pipelex.cogt.llm.llm_models.llm_engine import LLMEngine
from pipelex.cogt.llm.llm_models.llm_family import LLMFamily
from pipelex.cogt.llm.llm_models.llm_platform import LLMPlatform
from pipelex.cogt.llm.token_category import NbTokensByCategoryDict, TokenCategory, TokenCostsByCategoryDict
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.types import StrEnum
diff --git a/pipelex/cogt/llm/llm_worker_abstract.py b/pipelex/cogt/llm/llm_worker_abstract.py
index 7822ae72e..a9ed94be4 100644
--- a/pipelex/cogt/llm/llm_worker_abstract.py
+++ b/pipelex/cogt/llm/llm_worker_abstract.py
@@ -9,7 +9,7 @@ from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbst
from pipelex.cogt.llm.llm_job import LLMJob
from pipelex.cogt.llm.llm_models.llm_engine import LLMEngine
from pipelex.cogt.llm.structured_output import StructureMethod
-from pipelex.mission.job_metadata import UnitJobId
+from pipelex.pipeline.job_metadata import UnitJobId
from pipelex.tools.typing.pydantic_utils import BaseModelTypeVar
from pipelex.types import StrEnum
diff --git a/pipelex/cogt/ocr/ocr_job_factory.py b/pipelex/cogt/ocr/ocr_job_factory.py
index b6c2e4ce5..9220dd05a 100644
--- a/pipelex/cogt/ocr/ocr_job_factory.py
+++ b/pipelex/cogt/ocr/ocr_job_factory.py
@@ -4,7 +4,7 @@ from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job import OcrJob
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReport
from pipelex.config import get_config
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
class OcrJobFactory:
diff --git a/pipelex/cogt/ocr/ocr_worker_abstract.py b/pipelex/cogt/ocr/ocr_worker_abstract.py
index 982520cd1..6d5df0f18 100644
--- a/pipelex/cogt/ocr/ocr_worker_abstract.py
+++ b/pipelex/cogt/ocr/ocr_worker_abstract.py
@@ -10,7 +10,7 @@ from pipelex.cogt.inference.inference_worker_abstract import InferenceWorkerAbst
from pipelex.cogt.ocr.ocr_engine import OcrEngine
from pipelex.cogt.ocr.ocr_job import OcrJob
from pipelex.cogt.ocr.ocr_output import OcrOutput
-from pipelex.mission.job_metadata import UnitJobId
+from pipelex.pipeline.job_metadata import UnitJobId
F = TypeVar("F", bound=Callable[..., Awaitable[Any]])
diff --git a/pipelex/config.py b/pipelex/config.py
index d114b8804..7ecaa0fc6 100644
--- a/pipelex/config.py
+++ b/pipelex/config.py
@@ -7,7 +7,7 @@ from pipelex.cogt.llm.llm_models.llm_prompting_target import LLMPromptingTarget
from pipelex.exceptions import PipelexError
from pipelex.hub import get_required_config
from pipelex.libraries.library_config import LibraryConfig
-from pipelex.mission.track.tracker_config import TrackerConfig
+from pipelex.pipeline.track.tracker_config import TrackerConfig
from pipelex.plugins.plugins_config import PluginsConfig
from pipelex.tools.aws.aws_config import AwsConfig
from pipelex.tools.config.models import ConfigModel, ConfigRoot
@@ -44,7 +44,7 @@ class PromptingConfig(ConfigModel):
class FeatureConfig(ConfigModel):
- is_mission_tracking_enabled: bool
+ is_pipeline_tracking_enabled: bool
is_activity_tracking_enabled: bool
diff --git a/pipelex/core/pipe_abstract.py b/pipelex/core/pipe_abstract.py
index 51a01e9e2..df3d60069 100644
--- a/pipelex/core/pipe_abstract.py
+++ b/pipelex/core/pipe_abstract.py
@@ -7,7 +7,7 @@ from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeStackOverflowError
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeAbstract(ABC, BaseModel):
diff --git a/pipelex/exceptions.py b/pipelex/exceptions.py
index be6b4ec3f..715f3b6a1 100644
--- a/pipelex/exceptions.py
+++ b/pipelex/exceptions.py
@@ -167,5 +167,5 @@ class ConceptDomainError(ConceptError):
pass
-class MissionManagerNotFoundError(PipelexError):
+class PipelineManagerNotFoundError(PipelexError):
pass
diff --git a/pipelex/hub.py b/pipelex/hub.py
index 5c0905e98..95bf99731 100644
--- a/pipelex/hub.py
+++ b/pipelex/hub.py
@@ -18,11 +18,11 @@ from pipelex.core.domain import Domain
from pipelex.core.domain_provider_abstract import DomainProviderAbstract
from pipelex.core.pipe_abstract import PipeAbstract
from pipelex.core.pipe_provider_abstract import PipeProviderAbstract
-from pipelex.mission.activity.activity_manager_protocol import ActivityManagerProtocol
-from pipelex.mission.mission import Mission
-from pipelex.mission.mission_manager_abstract import MissionManagerAbstract
-from pipelex.mission.track.mission_tracker_protocol import MissionTrackerProtocol
from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
+from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol
+from pipelex.pipeline.pipeline import Pipeline
+from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract
+from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocol
from pipelex.tools.config.manager import config_manager
from pipelex.tools.config.models import ConfigRoot
from pipelex.tools.secrets.secrets_provider_abstract import SecretsProviderAbstract
@@ -57,9 +57,9 @@ class PipelexHub:
self._pipe_provider: Optional[PipeProviderAbstract] = None
self._pipe_router: Optional[PipeRouterProtocol] = None
- # mission
- self._mission_tracker: Optional[MissionTrackerProtocol] = None
- self._mission_manager: Optional[MissionManagerAbstract] = None
+ # pipeline
+ self._pipeline_tracker: Optional[PipelineTrackerProtocol] = None
+ self._pipeline_manager: Optional[PipelineManagerAbstract] = None
self._activity_manager: Optional[ActivityManagerProtocol] = None
############################################################
@@ -146,11 +146,11 @@ class PipelexHub:
def set_pipe_router(self, pipe_router: PipeRouterProtocol):
self._pipe_router = pipe_router
- def set_mission_tracker(self, mission_tracker: MissionTrackerProtocol):
- self._mission_tracker = mission_tracker
+ def set_pipeline_tracker(self, pipeline_tracker: PipelineTrackerProtocol):
+ self._pipeline_tracker = pipeline_tracker
- def set_mission_manager(self, mission_manager: MissionManagerAbstract):
- self._mission_manager = mission_manager
+ def set_pipeline_manager(self, pipeline_manager: PipelineManagerAbstract):
+ self._pipeline_manager = pipeline_manager
def set_activity_manager(self, activity_manager: ActivityManagerProtocol):
self._activity_manager = activity_manager
@@ -246,15 +246,15 @@ class PipelexHub:
raise RuntimeError("PipeRouter is not initialized")
return self._pipe_router
- def get_mission_tracker(self) -> MissionTrackerProtocol:
- if self._mission_tracker is None:
- raise RuntimeError("MissionTracker is not initialized")
- return self._mission_tracker
+ def get_pipeline_tracker(self) -> PipelineTrackerProtocol:
+ if self._pipeline_tracker is None:
+ raise RuntimeError("PipelineTracker is not initialized")
+ return self._pipeline_tracker
- def get_required_mission_manager(self) -> MissionManagerAbstract:
- if self._mission_manager is None:
- raise RuntimeError("MissionManager is not initialized")
- return self._mission_manager
+ def get_required_pipeline_manager(self) -> PipelineManagerAbstract:
+ if self._pipeline_manager is None:
+ raise RuntimeError("PipelineManager is not initialized")
+ return self._pipeline_manager
def get_activity_manager(self) -> ActivityManagerProtocol:
if self._activity_manager is None:
@@ -410,17 +410,17 @@ def get_pipe_router() -> PipeRouterProtocol:
return get_pipelex_hub().get_required_pipe_router()
-def get_mission_tracker() -> MissionTrackerProtocol:
- return get_pipelex_hub().get_mission_tracker()
+def get_pipeline_tracker() -> PipelineTrackerProtocol:
+ return get_pipelex_hub().get_pipeline_tracker()
-def get_mission_manager() -> MissionManagerAbstract:
- return get_pipelex_hub().get_required_mission_manager()
+def get_pipeline_manager() -> PipelineManagerAbstract:
+ return get_pipelex_hub().get_required_pipeline_manager()
def get_activity_manager() -> ActivityManagerProtocol:
return get_pipelex_hub().get_activity_manager()
-def get_mission(mission_id: str) -> Mission:
- return get_mission_manager().get_mission(mission_id=mission_id)
+def get_pipeline(pipeline_run_id: str) -> Pipeline:
+ return get_pipeline_manager().get_pipeline(pipeline_run_id=pipeline_run_id)
diff --git a/pipelex/mission/execute.py b/pipelex/mission/execute.py
deleted file mode 100644
index 483f53b9d..000000000
--- a/pipelex/mission/execute.py
+++ /dev/null
@@ -1,46 +0,0 @@
-from typing import Optional, Tuple
-
-from pipelex import pretty_print
-from pipelex.core.pipe_output import PipeOutput
-from pipelex.core.pipe_run_params import PipeOutputMultiplicity
-from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
-from pipelex.core.working_memory import WorkingMemory
-from pipelex.hub import get_mission_manager, get_pipe_router, get_report_delegate, get_required_pipe
-from pipelex.mission.job_metadata import JobMetadata
-from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
-
-
-async def execute_mission(
- pipe_code: str,
- working_memory: Optional[WorkingMemory] = None,
- output_name: Optional[str] = None,
- output_multiplicity: Optional[PipeOutputMultiplicity] = None,
- dynamic_output_concept_code: Optional[str] = None,
-) -> Tuple[PipeOutput, str]:
- mission = get_mission_manager().add_new_mission()
- mission_id = mission.mission_id
- get_report_delegate().open_registry(mission_id=mission_id)
- pipe = get_required_pipe(pipe_code=pipe_code)
-
- job_metadata = JobMetadata(
- mission_id=mission_id,
- )
-
- pipe_run_params = PipeRunParamsFactory.make_run_params(
- output_multiplicity=output_multiplicity,
- dynamic_output_concept_code=dynamic_output_concept_code,
- )
-
- pretty_print(pipe, title=f"Running pipe '{pipe_code}'")
- if working_memory:
- working_memory.pretty_print_summary()
-
- pipe_job = PipeJobFactory.make_pipe_job(
- pipe=pipe,
- pipe_run_params=pipe_run_params,
- job_metadata=job_metadata,
- working_memory=working_memory,
- output_name=output_name,
- )
-
- return await get_pipe_router().run_pipe_job(pipe_job), mission_id
diff --git a/pipelex/mission/mission.py b/pipelex/mission/mission.py
deleted file mode 100644
index 4ff33840e..000000000
--- a/pipelex/mission/mission.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from pydantic import BaseModel
-
-
-class Mission(BaseModel):
- mission_id: str
diff --git a/pipelex/mission/mission_factory.py b/pipelex/mission/mission_factory.py
deleted file mode 100644
index e293e72d4..000000000
--- a/pipelex/mission/mission_factory.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import shortuuid
-
-from pipelex.mission.mission import Mission
-
-
-class MissionFactory:
- @classmethod
- def make_mission(cls) -> Mission:
- return Mission(
- mission_id=shortuuid.uuid(),
- )
diff --git a/pipelex/mission/mission_manager.py b/pipelex/mission/mission_manager.py
deleted file mode 100644
index 4ca834966..000000000
--- a/pipelex/mission/mission_manager.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from typing import Dict, Optional
-
-from pydantic import Field, RootModel
-from typing_extensions import override
-
-from pipelex.exceptions import MissionManagerNotFoundError
-from pipelex.mission.mission import Mission
-from pipelex.mission.mission_factory import MissionFactory
-from pipelex.mission.mission_manager_abstract import MissionManagerAbstract
-
-MissionManagerRoot = Dict[str, Mission]
-
-
-class MissionManager(MissionManagerAbstract, RootModel[MissionManagerRoot]):
- root: MissionManagerRoot = Field(default_factory=dict)
-
- @override
- def setup(self):
- pass
-
- @override
- def teardown(self):
- self.root.clear()
-
- @override
- def get_optional_mission(self, mission_id: str) -> Optional[Mission]:
- return self.root.get(mission_id)
-
- @override
- def get_mission(self, mission_id: str) -> Mission:
- mission = self.get_optional_mission(mission_id=mission_id)
- if mission is None:
- raise MissionManagerNotFoundError(f"Mission {mission_id} not found")
- return mission
-
- def _set_mission(self, mission_id: str, mission: Mission) -> Mission:
- self.root[mission_id] = mission
- return mission
-
- @override
- def add_new_mission(self) -> Mission:
- mission = MissionFactory.make_mission()
- self._set_mission(mission_id=mission.mission_id, mission=mission)
- return mission
diff --git a/pipelex/pipe_controllers/pipe_batch.py b/pipelex/pipe_controllers/pipe_batch.py
index 4c0074632..e59ed65f7 100644
--- a/pipelex/pipe_controllers/pipe_batch.py
+++ b/pipelex/pipe_controllers/pipe_batch.py
@@ -14,8 +14,8 @@ from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeExecutionError
from pipelex.hub import get_pipe_router
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_controllers.pipe_controller import PipeController
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeBatch(PipeController):
@@ -122,7 +122,7 @@ class PipeBatch(PipeController):
# for branch_index, (required_stuff_list, item_input_stuff, item_output_stuff) in enumerate(
# zip(required_stuff_lists, item_stuffs, output_stuffs)
# ):
- # get_mission_tracker().add_batch_step(
+ # get_pipeline_tracker().add_batch_step(
# from_stuff=input_stuff,
# to_stuff=item_input_stuff,
# to_branch_index=branch_index,
@@ -130,7 +130,7 @@ class PipeBatch(PipeController):
# comment="PipeBatch.run_pipe() in zip",
# )
# for required_stuff in required_stuff_list:
- # get_mission_tracker().add_pipe_step(
+ # get_pipeline_tracker().add_pipe_step(
# from_stuff=required_stuff,
# to_stuff=item_output_stuff,
# pipe_code=self.branch_pipe_code,
@@ -142,7 +142,7 @@ class PipeBatch(PipeController):
# for branch_index, branch_output_stuff in enumerate(output_stuffs):
# branch_output_item_code = branch_output_item_codes[branch_index]
- # get_mission_tracker().add_aggregate_step(
+ # get_pipeline_tracker().add_aggregate_step(
# from_stuff=branch_output_stuff,
# to_stuff=output_stuff,
# pipe_layer=pipe_run_params.pipe_layers,
diff --git a/pipelex/pipe_controllers/pipe_condition.py b/pipelex/pipe_controllers/pipe_condition.py
index 61d1fad87..6bef2edfe 100644
--- a/pipelex/pipe_controllers/pipe_condition.py
+++ b/pipelex/pipe_controllers/pipe_condition.py
@@ -9,11 +9,11 @@ from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeConditionError, PipeDefinitionError, PipeExecutionError, PipeInputError, WorkingMemoryStuffNotFoundError
-from pipelex.hub import get_mission_tracker, get_pipe_router
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
+from pipelex.hub import get_pipe_router, get_pipeline_tracker
from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_operators.pipe_jinja2 import PipeJinja2, PipeJinja2Output
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list
@@ -130,7 +130,7 @@ class PipeCondition(PipeController):
raise PipeInputError(f"Some required stuff(s) not found - {error_details}") from exc
for required_stuff in required_stuffs:
- get_mission_tracker().add_condition_step(
+ get_pipeline_tracker().add_condition_step(
from_stuff=required_stuff,
to_condition=condition_details,
condition_expression=self.expression or self.applied_expression_jinja2,
@@ -146,7 +146,7 @@ class PipeCondition(PipeController):
pipe_run_params=pipe_run_params,
output_name=output_name,
)
- get_mission_tracker().add_choice_step(
+ get_pipeline_tracker().add_choice_step(
from_condition=condition_details,
to_stuff=pipe_output.main_stuff,
pipe_layer=pipe_run_params.pipe_layers,
diff --git a/pipelex/pipe_controllers/pipe_controller.py b/pipelex/pipe_controllers/pipe_controller.py
index d56456f13..ae2445bdf 100644
--- a/pipelex/pipe_controllers/pipe_controller.py
+++ b/pipelex/pipe_controllers/pipe_controller.py
@@ -7,7 +7,7 @@ from pipelex.core.pipe_abstract import PipeAbstract
from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeController(PipeAbstract):
diff --git a/pipelex/pipe_controllers/pipe_parallel.py b/pipelex/pipe_controllers/pipe_parallel.py
index b8c3d49bb..58d8025d2 100644
--- a/pipelex/pipe_controllers/pipe_parallel.py
+++ b/pipelex/pipe_controllers/pipe_parallel.py
@@ -11,10 +11,10 @@ from pipelex.core.stuff_content import StuffContent
from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeDefinitionError
-from pipelex.hub import get_mission_tracker
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.hub import get_pipeline_tracker
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sub_pipe import SubPipe
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeParallel(PipeController):
@@ -93,7 +93,7 @@ class PipeParallel(PipeController):
name=output_name,
)
for stuff in output_stuffs.values():
- get_mission_tracker().add_aggregate_step(
+ get_pipeline_tracker().add_aggregate_step(
from_stuff=stuff,
to_stuff=combined_output_stuff,
pipe_layer=pipe_run_params.pipe_layers,
diff --git a/pipelex/pipe_controllers/pipe_sequence.py b/pipelex/pipe_controllers/pipe_sequence.py
index 96c0ca86e..30e1f8c59 100644
--- a/pipelex/pipe_controllers/pipe_sequence.py
+++ b/pipelex/pipe_controllers/pipe_sequence.py
@@ -7,9 +7,9 @@ from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeRunParamsError
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_controllers.pipe_controller import PipeController
from pipelex.pipe_controllers.sub_pipe import SubPipe
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeSequence(PipeController):
diff --git a/pipelex/pipe_controllers/sub_pipe.py b/pipelex/pipe_controllers/sub_pipe.py
index 73f17e6f5..e8bf8c93a 100644
--- a/pipelex/pipe_controllers/sub_pipe.py
+++ b/pipelex/pipe_controllers/sub_pipe.py
@@ -7,10 +7,10 @@ from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import BatchParams, PipeOutputMultiplicity, PipeRunParams
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeInputError, WorkingMemoryStuffNotFoundError
-from pipelex.hub import get_mission_tracker, get_pipe_router, get_required_pipe
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_required_pipe
from pipelex.pipe_controllers.pipe_batch import PipeBatch
from pipelex.pipe_controllers.pipe_condition import PipeCondition
+from pipelex.pipeline.job_metadata import JobMetadata
# TODO: decide if SubPipe should be a PipeAbstract (it's probably the case)
@@ -79,7 +79,7 @@ class SubPipe(BaseModel):
)
new_output_stuff = pipe_output.main_stuff
for stuff in required_stuffs:
- get_mission_tracker().add_pipe_step(
+ get_pipeline_tracker().add_pipe_step(
from_stuff=stuff,
to_stuff=new_output_stuff,
pipe_code=self.pipe_code,
diff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.py
index 7b263e5c5..b805cc98a 100644
--- a/pipelex/pipe_operators/pipe_func.py
+++ b/pipelex/pipe_operators/pipe_func.py
@@ -8,8 +8,8 @@ from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.stuff_content import ListContent, StuffContent, TextContent
from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory import WorkingMemory
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_operators.pipe_operator import PipeOperator
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.func_registry import func_registry
diff --git a/pipelex/pipe_operators/pipe_img_gen.py b/pipelex/pipe_operators/pipe_img_gen.py
index 02907e9de..caabb0570 100644
--- a/pipelex/pipe_operators/pipe_img_gen.py
+++ b/pipelex/pipe_operators/pipe_img_gen.py
@@ -16,8 +16,8 @@ from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeDefinitionError, PipeInputError, PipeRunParamsError, WorkingMemoryStuffNotFoundError
from pipelex.hub import get_content_generator
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_operators.pipe_operator import PipeOperator
+from pipelex.pipeline.job_metadata import JobMetadata
# TODO: refacto this as part of the PipeImgGen blueprint/params
IMGG_PROMPT_NAME = "imgg_prompt"
diff --git a/pipelex/pipe_operators/pipe_jinja2.py b/pipelex/pipe_operators/pipe_jinja2.py
index 1807dfc89..11d6ea9ba 100644
--- a/pipelex/pipe_operators/pipe_jinja2.py
+++ b/pipelex/pipe_operators/pipe_jinja2.py
@@ -14,8 +14,8 @@ from pipelex.core.stuff_content import TextContent
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeDefinitionError, PipeRunParamsError
from pipelex.hub import get_content_generator, get_template, get_template_provider
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_operators.pipe_operator import PipeOperator
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.templating.jinja2_errors import Jinja2TemplateError
from pipelex.tools.templating.jinja2_parsing import check_jinja2_parsing
from pipelex.tools.templating.jinja2_required_variables import detect_jinja2_required_variables
diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.py
index 3c14954ba..951529fd3 100644
--- a/pipelex/pipe_operators/pipe_llm.py
+++ b/pipelex/pipe_operators/pipe_llm.py
@@ -35,11 +35,11 @@ from pipelex.hub import (
get_required_pipe,
get_template,
)
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
from pipelex.pipe_operators.pipe_jinja2_factory import PipeJinja2Factory
from pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutput
from pipelex.pipe_operators.pipe_operator import PipeOperator
from pipelex.pipe_operators.piped_llm_prompt_factory import PipedLLMPromptFactory
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
from pipelex.types import StrEnum
diff --git a/pipelex/pipe_operators/pipe_llm_prompt.py b/pipelex/pipe_operators/pipe_llm_prompt.py
index 750d4e525..87e6d0b4d 100644
--- a/pipelex/pipe_operators/pipe_llm_prompt.py
+++ b/pipelex/pipe_operators/pipe_llm_prompt.py
@@ -24,9 +24,9 @@ from pipelex.exceptions import (
WorkingMemoryTypeError,
)
from pipelex.hub import get_template
-from pipelex.mission.job_metadata import JobCategory, JobMetadata
from pipelex.pipe_operators.pipe_jinja2 import PipeJinja2, PipeJinja2Output
from pipelex.pipe_operators.pipe_operator import PipeOperator
+from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
from pipelex.tools.templating.templating_models import PromptingStyle
from pipelex.tools.typing.type_inspector import get_type_structure
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list, has_more_than_one_among_attributes_from_list
diff --git a/pipelex/pipe_operators/pipe_ocr.py b/pipelex/pipe_operators/pipe_ocr.py
index 4ed74042b..721b68f8f 100644
--- a/pipelex/pipe_operators/pipe_ocr.py
+++ b/pipelex/pipe_operators/pipe_ocr.py
@@ -15,8 +15,8 @@ from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory import WorkingMemory
from pipelex.exceptions import PipeDefinitionError
from pipelex.hub import get_content_generator
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_operators.pipe_operator import PipeOperator
+from pipelex.pipeline.job_metadata import JobMetadata
from pipelex.tools.pdf.pypdfium2_renderer import pypdfium2_renderer
from pipelex.tools.typing.validation_utils import has_exactly_one_among_attributes_from_list
diff --git a/pipelex/pipe_operators/pipe_operator.py b/pipelex/pipe_operators/pipe_operator.py
index 7b7b0d63e..68ac38318 100644
--- a/pipelex/pipe_operators/pipe_operator.py
+++ b/pipelex/pipe_operators/pipe_operator.py
@@ -8,8 +8,8 @@ from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
from pipelex.hub import get_activity_manager
-from pipelex.mission.activity.activity_models import ActivityReport
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.activity.activity_models import ActivityReport
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeOperator(PipeAbstract):
diff --git a/pipelex/pipe_operators/piped_llm_prompt_factory.py b/pipelex/pipe_operators/piped_llm_prompt_factory.py
index 4b43bcfb4..542b9d435 100644
--- a/pipelex/pipe_operators/piped_llm_prompt_factory.py
+++ b/pipelex/pipe_operators/piped_llm_prompt_factory.py
@@ -8,8 +8,8 @@ from pipelex.cogt.llm.llm_prompt_template_inputs import LLMPromptTemplateInputs
from pipelex.core.pipe_output import PipeOutput
from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.working_memory_factory import WorkingMemoryFactory
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_operators.pipe_llm_prompt import PipeLLMPrompt, PipeLLMPromptOutput
+from pipelex.pipeline.job_metadata import JobMetadata
class PipedLLMPromptFactory(LLMPromptFactoryAbstract):
diff --git a/pipelex/pipe_works/pipe_job.py b/pipelex/pipe_works/pipe_job.py
index 90bdf326b..6454c9577 100644
--- a/pipelex/pipe_works/pipe_job.py
+++ b/pipelex/pipe_works/pipe_job.py
@@ -5,7 +5,7 @@ from pydantic import BaseModel, Field
from pipelex.core.pipe_abstract import PipeAbstract
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeJob(BaseModel):
diff --git a/pipelex/pipe_works/pipe_job_factory.py b/pipelex/pipe_works/pipe_job_factory.py
index 30e75bf5e..c9e2f8801 100644
--- a/pipelex/pipe_works/pipe_job_factory.py
+++ b/pipelex/pipe_works/pipe_job_factory.py
@@ -4,8 +4,8 @@ from pipelex.core.pipe_abstract import PipeAbstract
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.working_memory import WorkingMemory
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_works.pipe_job import PipeJob
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeJobFactory:
diff --git a/pipelex/pipe_works/pipe_router.py b/pipelex/pipe_works/pipe_router.py
index 233a50326..9217367f0 100644
--- a/pipelex/pipe_works/pipe_router.py
+++ b/pipelex/pipe_works/pipe_router.py
@@ -7,9 +7,9 @@ from pipelex.core.pipe_output import PipeOutputType
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
from pipelex.hub import get_required_pipe
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from pipelex.pipe_works.pipe_router_protocol import PipeJob, PipeRouterProtocol
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeRouter(PipeRouterProtocol):
@@ -19,7 +19,7 @@ class PipeRouter(PipeRouterProtocol):
pipe_job: PipeJob,
wfid: Optional[str] = None,
) -> PipeOutputType: # pyright: ignore[reportInvalidTypeVarUse]
- log.debug(f"PipeRouter run_pipe_job: pipe_code={pipe_job.pipe.code}")
+ log.debug(f"Start run_pipe_job: pipe_code={pipe_job.pipe.code}")
working_memory = pipe_job.working_memory
pipe = pipe_job.pipe
@@ -31,6 +31,7 @@ class PipeRouter(PipeRouterProtocol):
output_name=pipe_job.output_name,
pipe_run_params=pipe_job.pipe_run_params,
)
+ log.debug(f"Completed run_pipe_job: pipe_code={pipe_job.pipe.code}")
return cast(PipeOutputType, pipe_output)
@override
@@ -43,7 +44,7 @@ class PipeRouter(PipeRouterProtocol):
output_name: Optional[str] = None,
wfid: Optional[str] = None,
) -> PipeOutputType: # pyright: ignore[reportInvalidTypeVarUse]
- log.debug(f"PipeRouter run_pipe_code: pipe_code={pipe_code}")
+ log.debug(f"Start run_pipe_code: pipe_code={pipe_code}")
pipe = get_required_pipe(pipe_code)
pipe_job = PipeJobFactory.make_pipe_job(
pipe=pipe,
@@ -56,4 +57,5 @@ class PipeRouter(PipeRouterProtocol):
pipe_job=pipe_job,
wfid=wfid,
)
+ log.debug(f"Completed run_pipe_code: pipe_code={pipe_code}")
return pipe_output
diff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.py
index df9b01a4e..bc9c4ea90 100644
--- a/pipelex/pipe_works/pipe_router_protocol.py
+++ b/pipelex/pipe_works/pipe_router_protocol.py
@@ -3,8 +3,8 @@ from typing import Optional, Protocol
from pipelex.core.pipe_output import PipeOutputType
from pipelex.core.pipe_run_params import PipeRunParams
from pipelex.core.working_memory import WorkingMemory
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_works.pipe_job import PipeJob
+from pipelex.pipeline.job_metadata import JobMetadata
class PipeRouterProtocol(Protocol):
diff --git a/pipelex/pipelex.py b/pipelex/pipelex.py
index 066918893..5a62d1c23 100644
--- a/pipelex/pipelex.py
+++ b/pipelex/pipelex.py
@@ -20,13 +20,13 @@ from pipelex.core.registry_models import PipelexRegistryModels
from pipelex.exceptions import PipelexConfigError, PipelexSetupError
from pipelex.hub import PipelexHub, set_pipelex_hub
from pipelex.libraries.library_manager import LibraryManager
-from pipelex.mission.activity.activity_manager import ActivityManager
-from pipelex.mission.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol
-from pipelex.mission.mission_manager import MissionManager
-from pipelex.mission.track.mission_tracker import MissionTracker
-from pipelex.mission.track.mission_tracker_protocol import MissionTrackerNoOp, MissionTrackerProtocol
from pipelex.pipe_works.pipe_router import PipeRouter
from pipelex.pipe_works.pipe_router_protocol import PipeRouterProtocol
+from pipelex.pipeline.activity.activity_manager import ActivityManager
+from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerNoOp, ActivityManagerProtocol
+from pipelex.pipeline.pipeline_manager import PipelineManager
+from pipelex.pipeline.track.pipeline_tracker import PipelineTracker
+from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerNoOp, PipelineTrackerProtocol
from pipelex.test_extras.registry_test_models import PipelexTestModels
from pipelex.tools.config.models import ConfigRoot
from pipelex.tools.func_registry import func_registry
@@ -53,8 +53,8 @@ class Pipelex:
llm_model_provider: Optional[LLMModelLibrary] = None,
plugin_manager: Optional[PluginManager] = None,
inference_manager: Optional[InferenceManager] = None,
- mission_manager: Optional[MissionManager] = None,
- mission_tracker: Optional[MissionTracker] = None,
+ pipeline_manager: Optional[PipelineManager] = None,
+ pipeline_tracker: Optional[PipelineTracker] = None,
activity_manager: Optional[ActivityManagerProtocol] = None,
) -> Self:
if cls._pipelex_instance is not None:
@@ -79,8 +79,8 @@ class Pipelex:
llm_model_provider: Optional[LLMModelLibrary] = None,
plugin_manager: Optional[PluginManager] = None,
inference_manager: Optional[InferenceManager] = None,
- mission_manager: Optional[MissionManager] = None,
- mission_tracker: Optional[MissionTracker] = None,
+ pipeline_manager: Optional[PipelineManager] = None,
+ pipeline_tracker: Optional[PipelineTracker] = None,
activity_manager: Optional[ActivityManagerProtocol] = None,
) -> None:
self.pipelex_hub = pipelex_hub or PipelexHub()
@@ -128,17 +128,17 @@ class Pipelex:
self.pipelex_hub.set_concept_provider(concept_provider=self.library_manager.concept_library)
self.pipelex_hub.set_pipe_provider(pipe_provider=self.library_manager.pipe_library)
- # pipelex mission
- self.mission_tracker: MissionTrackerProtocol
- if mission_tracker:
- self.mission_tracker = mission_tracker
- elif get_config().pipelex.feature_config.is_mission_tracking_enabled:
- self.mission_tracker = MissionTracker(tracker_config=get_config().pipelex.tracker_config)
+ # pipelex pipeline
+ self.pipeline_tracker: PipelineTrackerProtocol
+ if pipeline_tracker:
+ self.pipeline_tracker = pipeline_tracker
+ elif get_config().pipelex.feature_config.is_pipeline_tracking_enabled:
+ self.pipeline_tracker = PipelineTracker(tracker_config=get_config().pipelex.tracker_config)
else:
- self.mission_tracker = MissionTrackerNoOp()
- self.pipelex_hub.set_mission_tracker(mission_tracker=self.mission_tracker)
- self.mission_manager = mission_manager or MissionManager()
- self.pipelex_hub.set_mission_manager(mission_manager=self.mission_manager)
+ self.pipeline_tracker = PipelineTrackerNoOp()
+ self.pipelex_hub.set_pipeline_tracker(pipeline_tracker=self.pipeline_tracker)
+ self.pipeline_manager = pipeline_manager or PipelineManager()
+ self.pipelex_hub.set_pipeline_manager(pipeline_manager=self.pipeline_manager)
self.activity_manager: ActivityManagerProtocol
if activity_manager:
@@ -176,9 +176,9 @@ class Pipelex:
self.pipelex_hub.set_pipe_router(pipe_router or PipeRouter())
- # mission
- self.mission_tracker.setup()
- self.mission_manager.setup()
+ # pipeline
+ self.pipeline_tracker.setup()
+ self.pipeline_manager.setup()
log.debug(f"{PACKAGE_NAME} version {PACKAGE_VERSION} setup done for {get_config().project_name}")
@@ -206,8 +206,8 @@ class Pipelex:
def teardown(self):
# pipelex
- self.mission_manager.teardown()
- self.mission_tracker.teardown()
+ self.pipeline_manager.teardown()
+ self.pipeline_tracker.teardown()
self.library_manager.teardown()
self.template_provider.teardown()
self.activity_manager.teardown()
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index 1004cb6b8..a67c11ad8 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -5,7 +5,7 @@ extra_env_files = []
[pipelex.feature_config]
# WIP/Experimental feature flags
-is_mission_tracking_enabled = false
+is_pipeline_tracking_enabled = false
is_activity_tracking_enabled = false
####################################################################################################
diff --git a/pipelex/mission/activity/__init__.py b/pipelex/pipeline/activity/__init__.py
similarity index 100%
rename from pipelex/mission/activity/__init__.py
rename to pipelex/pipeline/activity/__init__.py
diff --git a/pipelex/mission/activity/activity_handler.py b/pipelex/pipeline/activity/activity_handler.py
similarity index 98%
rename from pipelex/mission/activity/activity_handler.py
rename to pipelex/pipeline/activity/activity_handler.py
index 4e072775a..904efc6bb 100644
--- a/pipelex/mission/activity/activity_handler.py
+++ b/pipelex/pipeline/activity/activity_handler.py
@@ -13,7 +13,7 @@ from pipelex.core.stuff_content import (
StructuredContent,
TextContent,
)
-from pipelex.mission.activity.activity_models import ActivityReport
+from pipelex.pipeline.activity.activity_models import ActivityReport
from pipelex.tools.misc.file_fetch_utils import fetch_file_from_url_httpx
from pipelex.tools.misc.file_utils import ensure_path, save_text_to_path
from pipelex.tools.misc.json_utils import save_as_json_to_path
diff --git a/pipelex/mission/activity/activity_manager.py b/pipelex/pipeline/activity/activity_manager.py
similarity index 87%
rename from pipelex/mission/activity/activity_manager.py
rename to pipelex/pipeline/activity/activity_manager.py
index 4f6f71ef9..b2ba80029 100644
--- a/pipelex/mission/activity/activity_manager.py
+++ b/pipelex/pipeline/activity/activity_manager.py
@@ -4,8 +4,8 @@ from pydantic import Field
from typing_extensions import override
from pipelex import log
-from pipelex.mission.activity.activity_manager_protocol import ActivityManagerProtocol
-from pipelex.mission.activity.activity_models import ActivityCallback, ActivityReport
+from pipelex.pipeline.activity.activity_manager_protocol import ActivityManagerProtocol
+from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReport
from pipelex.tools.exceptions import RootException
diff --git a/pipelex/mission/activity/activity_manager_protocol.py b/pipelex/pipeline/activity/activity_manager_protocol.py
similarity index 90%
rename from pipelex/mission/activity/activity_manager_protocol.py
rename to pipelex/pipeline/activity/activity_manager_protocol.py
index 81783b55e..a950b9281 100644
--- a/pipelex/mission/activity/activity_manager_protocol.py
+++ b/pipelex/pipeline/activity/activity_manager_protocol.py
@@ -2,7 +2,7 @@ from typing import Protocol
from typing_extensions import override
-from pipelex.mission.activity.activity_models import ActivityCallback, ActivityReport
+from pipelex.pipeline.activity.activity_models import ActivityCallback, ActivityReport
class ActivityManagerProtocol(Protocol):
diff --git a/pipelex/mission/activity/activity_models.py b/pipelex/pipeline/activity/activity_models.py
similarity index 78%
rename from pipelex/mission/activity/activity_models.py
rename to pipelex/pipeline/activity/activity_models.py
index e99c93bc5..afc8a37b9 100644
--- a/pipelex/mission/activity/activity_models.py
+++ b/pipelex/pipeline/activity/activity_models.py
@@ -2,7 +2,7 @@ from typing import Any, Callable
from pydantic import BaseModel
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
class ActivityReport(BaseModel):
diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.py
new file mode 100644
index 000000000..74a7ace42
--- /dev/null
+++ b/pipelex/pipeline/execute.py
@@ -0,0 +1,70 @@
+from typing import Optional, Tuple
+
+from pipelex import pretty_print
+from pipelex.core.pipe_output import PipeOutput
+from pipelex.core.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.working_memory import WorkingMemory
+from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe
+from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
+from pipelex.pipeline.job_metadata import JobMetadata
+
+
+async def execute_pipeline(
+ pipe_code: str,
+ working_memory: Optional[WorkingMemory] = None,
+ output_name: Optional[str] = None,
+ output_multiplicity: Optional[PipeOutputMultiplicity] = None,
+ dynamic_output_concept_code: Optional[str] = None,
+) -> Tuple[PipeOutput, str]:
+ """Execute a pipeline and wait for its completion.
+
+ This function executes a pipe and returns its output along with the pipeline run ID.
+ Unlike *start_pipeline*, this function waits for the pipe execution to complete
+ before returning, and it returns the output in addition to the pipeline run ID.
+
+ Parameters
+ ----------
+ pipe_code:
+ The code of the pipe to execute.
+ working_memory:
+ Optional ``WorkingMemory`` instance passed to the pipe.
+ output_name:
+ Name of the output slot to write to.
+ output_multiplicity:
+ Output multiplicity.
+ dynamic_output_concept_code:
+ Override the dynamic output concept code.
+
+ Returns
+ -------
+ Tuple[PipeOutput, str]
+ A tuple containing the pipe output and the pipeline run ID.
+ """
+ pipeline = get_pipeline_manager().add_new_pipeline()
+ pipeline_run_id = pipeline.pipeline_run_id
+ get_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)
+ pipe = get_required_pipe(pipe_code=pipe_code)
+
+ job_metadata = JobMetadata(
+ pipeline_run_id=pipeline_run_id,
+ )
+
+ pipe_run_params = PipeRunParamsFactory.make_run_params(
+ output_multiplicity=output_multiplicity,
+ dynamic_output_concept_code=dynamic_output_concept_code,
+ )
+
+ pretty_print(pipe, title=f"Running pipe '{pipe_code}'")
+ if working_memory:
+ working_memory.pretty_print_summary()
+
+ pipe_job = PipeJobFactory.make_pipe_job(
+ pipe=pipe,
+ pipe_run_params=pipe_run_params,
+ job_metadata=job_metadata,
+ working_memory=working_memory,
+ output_name=output_name,
+ )
+
+ return await get_pipe_router().run_pipe_job(pipe_job), pipeline_run_id
diff --git a/pipelex/mission/job_metadata.py b/pipelex/pipeline/job_metadata.py
similarity index 94%
rename from pipelex/mission/job_metadata.py
rename to pipelex/pipeline/job_metadata.py
index ee74a0d98..104d60a2b 100644
--- a/pipelex/mission/job_metadata.py
+++ b/pipelex/pipeline/job_metadata.py
@@ -3,7 +3,7 @@ from typing import List, Optional
from pydantic import BaseModel, ConfigDict, Field
-from pipelex.mission.mission_models import SpecialMissionId
+from pipelex.pipeline.pipeline_models import SpecialPipelineId
from pipelex.types import StrEnum
@@ -26,7 +26,7 @@ class UnitJobId(StrEnum):
class JobMetadata(BaseModel):
model_config = ConfigDict(arbitrary_types_allowed=True)
- mission_id: str = Field(default=SpecialMissionId.UNTITLED)
+ pipeline_run_id: str = Field(default=SpecialPipelineId.UNTITLED)
top_job_id: Optional[str] = None
pipe_job_ids: Optional[List[str]] = None
content_generation_job_id: Optional[str] = None
diff --git a/pipelex/pipeline/pipeline.py b/pipelex/pipeline/pipeline.py
new file mode 100644
index 000000000..c16495476
--- /dev/null
+++ b/pipelex/pipeline/pipeline.py
@@ -0,0 +1,5 @@
+from pydantic import BaseModel
+
+
+class Pipeline(BaseModel):
+ pipeline_run_id: str
diff --git a/pipelex/pipeline/pipeline_factory.py b/pipelex/pipeline/pipeline_factory.py
new file mode 100644
index 000000000..e56931907
--- /dev/null
+++ b/pipelex/pipeline/pipeline_factory.py
@@ -0,0 +1,11 @@
+import shortuuid
+
+from pipelex.pipeline.pipeline import Pipeline
+
+
+class PipelineFactory:
+ @classmethod
+ def make_pipeline(cls) -> Pipeline:
+ return Pipeline(
+ pipeline_run_id=shortuuid.uuid(),
+ )
diff --git a/pipelex/pipeline/pipeline_manager.py b/pipelex/pipeline/pipeline_manager.py
new file mode 100644
index 000000000..99600a1c0
--- /dev/null
+++ b/pipelex/pipeline/pipeline_manager.py
@@ -0,0 +1,44 @@
+from typing import Dict, Optional
+
+from pydantic import Field, RootModel
+from typing_extensions import override
+
+from pipelex.exceptions import PipelineManagerNotFoundError
+from pipelex.pipeline.pipeline import Pipeline
+from pipelex.pipeline.pipeline_factory import PipelineFactory
+from pipelex.pipeline.pipeline_manager_abstract import PipelineManagerAbstract
+
+PipelineManagerRoot = Dict[str, Pipeline]
+
+
+class PipelineManager(PipelineManagerAbstract, RootModel[PipelineManagerRoot]):
+ root: PipelineManagerRoot = Field(default_factory=dict)
+
+ @override
+ def setup(self):
+ pass
+
+ @override
+ def teardown(self):
+ self.root.clear()
+
+ @override
+ def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:
+ return self.root.get(pipeline_run_id)
+
+ @override
+ def get_pipeline(self, pipeline_run_id: str) -> Pipeline:
+ pipeline = self.get_optional_pipeline(pipeline_run_id=pipeline_run_id)
+ if pipeline is None:
+ raise PipelineManagerNotFoundError(f"Pipeline {pipeline_run_id} not found")
+ return pipeline
+
+ def _set_pipeline(self, pipeline_run_id: str, pipeline: Pipeline) -> Pipeline:
+ self.root[pipeline_run_id] = pipeline
+ return pipeline
+
+ @override
+ def add_new_pipeline(self) -> Pipeline:
+ pipeline = PipelineFactory.make_pipeline()
+ self._set_pipeline(pipeline_run_id=pipeline.pipeline_run_id, pipeline=pipeline)
+ return pipeline
diff --git a/pipelex/mission/mission_manager_abstract.py b/pipelex/pipeline/pipeline_manager_abstract.py
similarity index 52%
rename from pipelex/mission/mission_manager_abstract.py
rename to pipelex/pipeline/pipeline_manager_abstract.py
index dfc69684d..1c7e13878 100644
--- a/pipelex/mission/mission_manager_abstract.py
+++ b/pipelex/pipeline/pipeline_manager_abstract.py
@@ -1,10 +1,10 @@
from abc import ABC, abstractmethod
from typing import Optional
-from pipelex.mission.mission import Mission
+from pipelex.pipeline.pipeline import Pipeline
-class MissionManagerAbstract(ABC):
+class PipelineManagerAbstract(ABC):
@abstractmethod
def setup(self) -> None:
pass
@@ -14,13 +14,13 @@ class MissionManagerAbstract(ABC):
pass
@abstractmethod
- def get_optional_mission(self, mission_id: str) -> Optional[Mission]:
+ def get_optional_pipeline(self, pipeline_run_id: str) -> Optional[Pipeline]:
pass
@abstractmethod
- def get_mission(self, mission_id: str) -> Mission:
+ def get_pipeline(self, pipeline_run_id: str) -> Pipeline:
pass
@abstractmethod
- def add_new_mission(self) -> Mission:
+ def add_new_pipeline(self) -> Pipeline:
pass
diff --git a/pipelex/mission/mission_models.py b/pipelex/pipeline/pipeline_models.py
similarity index 64%
rename from pipelex/mission/mission_models.py
rename to pipelex/pipeline/pipeline_models.py
index 4335f69b2..5fd4308e6 100644
--- a/pipelex/mission/mission_models.py
+++ b/pipelex/pipeline/pipeline_models.py
@@ -1,5 +1,5 @@
from pipelex.types import StrEnum
-class SpecialMissionId(StrEnum):
+class SpecialPipelineId(StrEnum):
UNTITLED = "untitled"
diff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.py
new file mode 100644
index 000000000..022539847
--- /dev/null
+++ b/pipelex/pipeline/start.py
@@ -0,0 +1,77 @@
+import asyncio
+from typing import Optional, Tuple
+
+from pipelex import pretty_print
+from pipelex.core.pipe_output import PipeOutput
+from pipelex.core.pipe_run_params import PipeOutputMultiplicity
+from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
+from pipelex.core.working_memory import WorkingMemory
+from pipelex.hub import get_pipe_router, get_pipeline_manager, get_report_delegate, get_required_pipe
+from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
+from pipelex.pipeline.job_metadata import JobMetadata
+
+
+async def start_pipeline(
+ pipe_code: str,
+ working_memory: Optional[WorkingMemory] = None,
+ output_name: Optional[str] = None,
+ output_multiplicity: Optional[PipeOutputMultiplicity] = None,
+ dynamic_output_concept_code: Optional[str] = None,
+) -> Tuple[str, asyncio.Task[PipeOutput]]:
+ """Start a pipeline in the background.
+
+ This function mirrors *execute_pipeline* but returns immediately with the
+ ``pipeline_run_id`` instead of waiting for the pipe run to complete. The
+ actual execution is scheduled on the current event-loop using
+ :pyfunc:`asyncio.create_task`.
+
+ Parameters
+ ----------
+ pipe_code:
+ The code of the pipe to execute.
+ working_memory:
+ Optional ``WorkingMemory`` instance passed to the pipe.
+ output_name:
+ Name of the output slot to write to.
+ output_multiplicity:
+ Output multiplicity.
+ dynamic_output_concept_code:
+ Override the dynamic output concept code.
+
+ Returns
+ -------
+ Tuple[str, asyncio.Task[PipeOutput]]
+ The ``pipeline_run_id`` of the newly started pipeline and a task that
+ can be awaited to get the pipe output.
+ """
+
+ pipeline = get_pipeline_manager().add_new_pipeline()
+ pipeline_run_id = pipeline.pipeline_run_id
+ get_report_delegate().open_registry(pipeline_run_id=pipeline_run_id)
+ pipe = get_required_pipe(pipe_code=pipe_code)
+
+ job_metadata = JobMetadata(
+ pipeline_run_id=pipeline_run_id,
+ )
+
+ pipe_run_params = PipeRunParamsFactory.make_run_params(
+ output_multiplicity=output_multiplicity,
+ dynamic_output_concept_code=dynamic_output_concept_code,
+ )
+
+ pretty_print(pipe, title=f"Starting pipe '{pipe_code}' (background)")
+ if working_memory:
+ working_memory.pretty_print_summary()
+
+ pipe_job = PipeJobFactory.make_pipe_job(
+ pipe=pipe,
+ pipe_run_params=pipe_run_params,
+ job_metadata=job_metadata,
+ working_memory=working_memory,
+ output_name=output_name,
+ )
+
+ # Launch execution without awaiting the result.
+ task: asyncio.Task[PipeOutput] = asyncio.create_task(get_pipe_router().run_pipe_job(pipe_job))
+
+ return pipeline_run_id, task
diff --git a/pipelex/mission/track/__init__.py b/pipelex/pipeline/track/__init__.py
similarity index 100%
rename from pipelex/mission/track/__init__.py
rename to pipelex/pipeline/track/__init__.py
diff --git a/pipelex/mission/track/flow_chart.py b/pipelex/pipeline/track/flow_chart.py
similarity index 98%
rename from pipelex/mission/track/flow_chart.py
rename to pipelex/pipeline/track/flow_chart.py
index 856c12dd8..6736fb75e 100644
--- a/pipelex/mission/track/flow_chart.py
+++ b/pipelex/pipeline/track/flow_chart.py
@@ -10,8 +10,8 @@ import yaml
from pipelex import log
from pipelex.exceptions import JobHistoryError
-from pipelex.mission.track.tracker_config import TrackerConfig
-from pipelex.mission.track.tracker_models import (
+from pipelex.pipeline.track.tracker_config import TrackerConfig
+from pipelex.pipeline.track.tracker_models import (
EdgeAttributeKey,
EdgeCategory,
GraphTree,
@@ -31,7 +31,7 @@ def _indent_line(line: str, indent: int) -> str:
return f"{' ' * indent}{line}"
-class MissionFlowChart:
+class PipelineFlowChart:
def __init__(self, nx_graph: nx.DiGraph, start_node: str, tracker_config: TrackerConfig):
self.nx_graph = nx_graph
self._tracker_config = tracker_config
diff --git a/pipelex/mission/track/mission_tracker.py b/pipelex/pipeline/track/pipeline_tracker.py
similarity index 94%
rename from pipelex/mission/track/mission_tracker.py
rename to pipelex/pipeline/track/pipeline_tracker.py
index 832aa529e..0486e4719 100644
--- a/pipelex/mission/track/mission_tracker.py
+++ b/pipelex/pipeline/track/pipeline_tracker.py
@@ -11,23 +11,23 @@ from pipelex import log
from pipelex.core.concept import Concept
from pipelex.core.stuff import Stuff
from pipelex.exceptions import JobHistoryError
-from pipelex.mission.track.flow_chart import MissionFlowChart
-from pipelex.mission.track.mission_tracker_protocol import MissionTrackerProtocol
-from pipelex.mission.track.tracker_config import TrackerConfig
-from pipelex.mission.track.tracker_models import (
+from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails
+from pipelex.pipeline.track.flow_chart import PipelineFlowChart
+from pipelex.pipeline.track.pipeline_tracker_protocol import PipelineTrackerProtocol
+from pipelex.pipeline.track.tracker_config import TrackerConfig
+from pipelex.pipeline.track.tracker_models import (
EdgeAttributeKey,
EdgeCategory,
NodeAttributeKey,
NodeCategory,
SpecialNodeName,
)
-from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails
from pipelex.tools.misc.mermaid_utils import print_mermaid_url
-# TODO: manage a separate graph for each mission_id
+# TODO: manage a separate graph for each pipeline_run_id
# TODO: restore disabled tracking functionality in PipeBatch
-class MissionTracker(MissionTrackerProtocol):
+class PipelineTracker(PipelineTrackerProtocol):
def __init__(self, tracker_config: TrackerConfig):
self._tracker_config = tracker_config
self._is_debug_mode = tracker_config.is_debug_mode
@@ -320,11 +320,11 @@ class MissionTracker(MissionTrackerProtocol):
def _print_mermaid_flowchart_code_and_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):
if not self.nx_graph.nodes:
- log.info("No nodes in the mission tracker")
+ log.info("No nodes in the pipeline tracker")
return
if self.start_node is None:
raise JobHistoryError("Start node is not set")
- flowchart = MissionFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
+ flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
mermaid_code, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)
print(mermaid_code)
title_to_print = "Mermaid flowchart URL"
@@ -334,11 +334,11 @@ class MissionTracker(MissionTrackerProtocol):
def _print_mermaid_flowchart_url(self, title: Optional[str] = None, subtitle: Optional[str] = None):
if not self.nx_graph.nodes:
- log.info("No nodes in the mission tracker")
+ log.info("No nodes in the pipeline tracker")
return
if self.start_node is None:
raise JobHistoryError("Start node is not set")
- flowchart = MissionFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
+ flowchart = PipelineFlowChart(nx_graph=self.nx_graph, start_node=self.start_node, tracker_config=self._tracker_config)
_, url = flowchart.generate_mermaid_flowchart(title=title, subtitle=subtitle)
title_to_print = "Mermaid flowchart URL"
if title:
diff --git a/pipelex/mission/track/mission_tracker_protocol.py b/pipelex/pipeline/track/pipeline_tracker_protocol.py
similarity index 92%
rename from pipelex/mission/track/mission_tracker_protocol.py
rename to pipelex/pipeline/track/pipeline_tracker_protocol.py
index f71400134..95cb08528 100644
--- a/pipelex/mission/track/mission_tracker_protocol.py
+++ b/pipelex/pipeline/track/pipeline_tracker_protocol.py
@@ -10,7 +10,7 @@ from pipelex.core.stuff import Stuff
from pipelex.pipe_controllers.pipe_condition_details import PipeConditionDetails
-class MissionTrackerProtocol(Protocol):
+class PipelineTrackerProtocol(Protocol):
def setup(self): ...
def teardown(self): ...
@@ -69,9 +69,9 @@ class MissionTrackerProtocol(Protocol):
): ...
-class MissionTrackerNoOp(MissionTrackerProtocol):
- """A no-operation implementation of MissionTrackerProtocol that does nothing.
- This is useful when mission tracking needs to be disabled.
+class PipelineTrackerNoOp(PipelineTrackerProtocol):
+ """A no-operation implementation of PipelineTrackerProtocol that does nothing.
+ This is useful when pipeline tracking needs to be disabled.
"""
@override
diff --git a/pipelex/mission/track/tracker_config.py b/pipelex/pipeline/track/tracker_config.py
similarity index 100%
rename from pipelex/mission/track/tracker_config.py
rename to pipelex/pipeline/track/tracker_config.py
diff --git a/pipelex/mission/track/tracker_models.py b/pipelex/pipeline/track/tracker_models.py
similarity index 100%
rename from pipelex/mission/track/tracker_models.py
rename to pipelex/pipeline/track/tracker_models.py
diff --git a/pipelex/run.py b/pipelex/run.py
index 1c3614e05..2b2924204 100644
--- a/pipelex/run.py
+++ b/pipelex/run.py
@@ -7,8 +7,8 @@ from pipelex.core.pipe_run_params import PipeOutputMultiplicity, PipeRunParams
from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.working_memory import WorkingMemory
from pipelex.hub import get_pipe_router, get_required_pipe
-from pipelex.mission.job_metadata import JobMetadata
from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
+from pipelex.pipeline.job_metadata import JobMetadata
async def run_pipe_code(
diff --git a/tests/pipelex/cogt/cogt_asynch/test_content_generator.py b/tests/pipelex/cogt/cogt_asynch/test_content_generator.py
index c6945d651..37503d0a5 100644
--- a/tests/pipelex/cogt/cogt_asynch/test_content_generator.py
+++ b/tests/pipelex/cogt/cogt_asynch/test_content_generator.py
@@ -15,7 +15,7 @@ from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams
from pipelex.cogt.ocr.ocr_output import OcrOutput
from pipelex.hub import get_content_generator, get_llm_deck
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.pipeline.job_metadata import JobMetadata
from tests.pipelex.cogt.test_data import Employee
from tests.test_data import ImageTestCases
diff --git a/tests/pipelex/pipelex_asynch/conftest.py b/tests/pipelex/pipelex_asynch/conftest.py
index 0b78a8552..dc04dd01f 100644
--- a/tests/pipelex/pipelex_asynch/conftest.py
+++ b/tests/pipelex/pipelex_asynch/conftest.py
@@ -6,7 +6,7 @@ from pytest import FixtureRequest
from pipelex.core.pipe_output import PipeOutput
from pipelex.hub import get_activity_manager
-from pipelex.mission.activity.activity_handler import ActivityHandlerForResultFiles
+from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
from pipelex.tools.misc.file_utils import get_incremental_directory_path, remove_folder
from pipelex.tools.misc.json_utils import save_as_json_to_path
from tests.conftest import TEST_OUTPUTS_DIR
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_batch.py b/tests/pipelex/pipelex_asynch/test_pipe_batch.py
index 4adac5ddb..10fae5f2f 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_batch.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_batch.py
@@ -6,7 +6,7 @@ from pipelex.core.pipe_run_params_factory import PipeRunParamsFactory
from pipelex.core.stuff_content import ListContent, TextContent
from pipelex.core.stuff_factory import StuffFactory
from pipelex.core.working_memory_factory import WorkingMemoryFactory
-from pipelex.hub import get_mission_tracker, get_pipe_router, get_report_delegate
+from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_report_delegate
@pytest.mark.llm
@@ -45,4 +45,4 @@ class TestPipeBatch:
assert pipe_output.working_memory is not None
assert pipe_output.main_stuff is not None
- get_mission_tracker().output_flowchart()
+ get_pipeline_tracker().output_flowchart()
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py
index 2d7b4bf8c..2bdcf796d 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py
@@ -11,9 +11,9 @@ from pipelex.core.stuff import Stuff
from pipelex.core.stuff_factory import StuffBlueprint
from pipelex.core.working_memory import WorkingMemory
from pipelex.core.working_memory_factory import WorkingMemoryFactory
-from pipelex.hub import get_mission_tracker, get_pipe_router, get_report_delegate
-from pipelex.mission.activity.activity_handler import ActivityHandlerForResultFiles
-from pipelex.mission.job_metadata import JobMetadata
+from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_report_delegate
+from pipelex.pipeline.activity.activity_handler import ActivityHandlerForResultFiles
+from pipelex.pipeline.job_metadata import JobMetadata
from tests.pipelex.test_data import PipeTestCases
@@ -166,7 +166,7 @@ class TestPipeRouter:
pretty_print(pipe_output, title=f"run pipe '{pipe_code}'")
get_report_delegate().generate_report()
- get_mission_tracker().output_flowchart(is_detailed=True)
+ get_pipeline_tracker().output_flowchart(is_detailed=True)
@pytest.mark.parametrize("pipe_code, exception, expected_error_message", PipeTestCases.FAILURE_PIPES)
async def test_pipe_infinite_loop(