Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

fix/pipeline_run_id-in-JobMetadata (#121)

Thomas Hebrard9378a60c0415
diff --git a/.cursor/rules/pytest.mdc b/.cursor/rules/pytest.mdc
index 0a4b8d075..32b4b39ce 100644
--- a/.cursor/rules/pytest.mdc
+++ b/.cursor/rules/pytest.mdc
@@ -104,10 +104,7 @@ pipe_output: PipeOutput = await pipe_router.run_pipe(
pipe_code="pipe_name",
pipe_run_params=PipeRunParamsFactory.make_run_params(),
working_memory=working_memory,
- job_metadata=JobMetadata(
-
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(),
)
```
diff --git a/pipelex/cogt/content_generation/content_generator.py b/pipelex/cogt/content_generation/content_generator.py
index 4c1c8a884..da1353f12 100644
--- a/pipelex/cogt/content_generation/content_generator.py
+++ b/pipelex/cogt/content_generation/content_generator.py
@@ -44,7 +44,6 @@ class ContentGenerator(ContentGeneratorProtocol):
job_metadata: JobMetadata,
llm_setting_main: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- wfid: Optional[str] = None,
) -> str:
log.verbose(f"{self.__class__.__name__} make_llm_text: {llm_prompt_for_text}")
log.verbose(f"llm_setting_main: {llm_setting_main}")
@@ -66,7 +65,6 @@ class ContentGenerator(ContentGeneratorProtocol):
object_class: Type[BaseModelTypeVar],
llm_setting_for_object: LLMSetting,
llm_prompt_for_object: LLMPrompt,
- wfid: Optional[str] = None,
) -> BaseModelTypeVar:
log.verbose(f"{self.__class__.__name__} make_object_direct: {llm_prompt_for_object}")
llm_assignment_for_object = LLMAssignment(
@@ -92,7 +90,6 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_setting_for_object: LLMSetting,
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
- wfid: Optional[str] = None,
) -> BaseModelTypeVar:
llm_assignment_for_text = LLMAssignment(
job_metadata=job_metadata,
@@ -137,7 +134,6 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
nb_items: Optional[int] = None,
- wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
llm_assignment_for_object = LLMAssignment(
job_metadata=job_metadata,
@@ -163,7 +159,6 @@ class ContentGenerator(ContentGeneratorProtocol):
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
nb_items: Optional[int] = None,
- wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
llm_assignment_for_text = LLMAssignment(
job_metadata=job_metadata,
@@ -208,7 +203,6 @@ class ContentGenerator(ContentGeneratorProtocol):
imgg_prompt: ImggPrompt,
imgg_job_params: Optional[ImggJobParams] = None,
imgg_job_config: Optional[ImggJobConfig] = None,
- wfid: Optional[str] = None,
) -> GeneratedImage:
imgg_config = get_config().cogt.imgg_config
imgg_assignment = ImggAssignment(
@@ -233,7 +227,6 @@ class ContentGenerator(ContentGeneratorProtocol):
nb_images: int,
imgg_job_params: Optional[ImggJobParams] = None,
imgg_job_config: Optional[ImggJobConfig] = None,
- wfid: Optional[str] = None,
) -> List[GeneratedImage]:
imgg_config = get_config().cogt.imgg_config
imgg_assignment = ImggAssignment(
@@ -256,7 +249,6 @@ class ContentGenerator(ContentGeneratorProtocol):
jinja2: Optional[str] = None,
prompting_style: Optional[PromptingStyle] = None,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
- wfid: Optional[str] = None,
) -> str:
jinja2_assignment = Jinja2Assignment(
context=context,
@@ -277,7 +269,6 @@ class ContentGenerator(ContentGeneratorProtocol):
ocr_handle: OcrHandle,
ocr_job_params: Optional[OcrJobParams] = None,
ocr_job_config: Optional[OcrJobConfig] = None,
- wfid: Optional[str] = None,
) -> OcrOutput:
ocr_assignment = OcrAssignment(
job_metadata=job_metadata,
diff --git a/pipelex/cogt/content_generation/content_generator_dry.py b/pipelex/cogt/content_generation/content_generator_dry.py
index e97509026..6cfc66109 100644
--- a/pipelex/cogt/content_generation/content_generator_dry.py
+++ b/pipelex/cogt/content_generation/content_generator_dry.py
@@ -40,7 +40,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
job_metadata: JobMetadata,
llm_setting_main: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- wfid: Optional[str] = None,
) -> str:
func_name = "make_llm_text"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -56,7 +55,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
object_class: Type[BaseModelTypeVar],
llm_setting_for_object: LLMSetting,
llm_prompt_for_object: LLMPrompt,
- wfid: Optional[str] = None,
) -> BaseModelTypeVar:
func_name = "make_object_direct"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -78,7 +76,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
llm_setting_for_object: LLMSetting,
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
- wfid: Optional[str] = None,
) -> BaseModelTypeVar:
func_name = "make_text_then_object"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -98,7 +95,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
nb_items: Optional[int] = None,
- wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
func_name = "make_object_list_direct"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -125,7 +121,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
nb_items: Optional[int] = None,
- wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]:
func_name = "make_text_then_object_list"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -146,7 +141,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
imgg_prompt: ImggPrompt,
imgg_job_params: Optional[ImggJobParams] = None,
imgg_job_config: Optional[ImggJobConfig] = None,
- wfid: Optional[str] = None,
) -> GeneratedImage:
func_name = "make_single_image"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -169,7 +163,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
nb_images: int,
imgg_job_params: Optional[ImggJobParams] = None,
imgg_job_config: Optional[ImggJobConfig] = None,
- wfid: Optional[str] = None,
) -> List[GeneratedImage]:
func_name = "make_image_list"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -192,7 +185,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
jinja2: Optional[str] = None,
prompting_style: Optional[PromptingStyle] = None,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
- wfid: Optional[str] = None,
) -> str:
func_name = "make_jinja2_text"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
@@ -211,7 +203,6 @@ class ContentGeneratorDry(ContentGeneratorProtocol):
ocr_handle: OcrHandle,
ocr_job_params: Optional[OcrJobParams] = None,
ocr_job_config: Optional[OcrJobConfig] = None,
- wfid: Optional[str] = None,
) -> OcrOutput:
func_name = "make_ocr_extract_pages"
log.dev(f"🤡 DRY RUN: {self.__class__.__name__}.{func_name}")
diff --git a/pipelex/cogt/content_generation/content_generator_protocol.py b/pipelex/cogt/content_generation/content_generator_protocol.py
index 6945fe22e..2f1cfd4cf 100644
--- a/pipelex/cogt/content_generation/content_generator_protocol.py
+++ b/pipelex/cogt/content_generation/content_generator_protocol.py
@@ -49,7 +49,6 @@ class ContentGeneratorProtocol(Protocol):
job_metadata: JobMetadata,
llm_setting_main: LLMSetting,
llm_prompt_for_text: LLMPrompt,
- wfid: Optional[str] = None,
) -> str: ...
async def make_object_direct(
@@ -58,7 +57,6 @@ class ContentGeneratorProtocol(Protocol):
object_class: Type[BaseModelTypeVar],
llm_setting_for_object: LLMSetting,
llm_prompt_for_object: LLMPrompt,
- wfid: Optional[str] = None,
) -> BaseModelTypeVar: ...
async def make_text_then_object(
@@ -69,7 +67,6 @@ class ContentGeneratorProtocol(Protocol):
llm_setting_for_object: LLMSetting,
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object: Optional[LLMPromptFactoryAbstract] = None,
- wfid: Optional[str] = None,
) -> BaseModelTypeVar: ...
async def make_object_list_direct(
@@ -79,7 +76,6 @@ class ContentGeneratorProtocol(Protocol):
llm_setting_for_object_list: LLMSetting,
llm_prompt_for_object_list: LLMPrompt,
nb_items: Optional[int] = None,
- wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]: ...
async def make_text_then_object_list(
@@ -91,7 +87,6 @@ class ContentGeneratorProtocol(Protocol):
llm_prompt_for_text: LLMPrompt,
llm_prompt_factory_for_object_list: Optional[LLMPromptFactoryAbstract] = None,
nb_items: Optional[int] = None,
- wfid: Optional[str] = None,
) -> List[BaseModelTypeVar]: ...
async def make_single_image(
@@ -101,7 +96,6 @@ class ContentGeneratorProtocol(Protocol):
imgg_prompt: ImggPrompt,
imgg_job_params: Optional[ImggJobParams] = None,
imgg_job_config: Optional[ImggJobConfig] = None,
- wfid: Optional[str] = None,
) -> GeneratedImage: ...
async def make_image_list(
@@ -112,7 +106,6 @@ class ContentGeneratorProtocol(Protocol):
nb_images: int,
imgg_job_params: Optional[ImggJobParams] = None,
imgg_job_config: Optional[ImggJobConfig] = None,
- wfid: Optional[str] = None,
) -> List[GeneratedImage]: ...
async def make_jinja2_text(
@@ -122,7 +115,6 @@ class ContentGeneratorProtocol(Protocol):
jinja2: Optional[str] = None,
prompting_style: Optional[PromptingStyle] = None,
template_category: Jinja2TemplateCategory = Jinja2TemplateCategory.LLM_PROMPT,
- wfid: Optional[str] = None,
) -> str: ...
async def make_ocr_extract_pages(
@@ -132,5 +124,4 @@ class ContentGeneratorProtocol(Protocol):
ocr_handle: OcrHandle,
ocr_job_params: OcrJobParams,
ocr_job_config: OcrJobConfig,
- wfid: Optional[str] = None,
) -> OcrOutput: ...
diff --git a/pipelex/cogt/imgg/imgg_job_factory.py b/pipelex/cogt/imgg/imgg_job_factory.py
index 91fbb8587..e5e7c1fc7 100644
--- a/pipelex/cogt/imgg/imgg_job_factory.py
+++ b/pipelex/cogt/imgg/imgg_job_factory.py
@@ -18,10 +18,8 @@ class ImggJobFactory:
imgg_job_config: Optional[ImggJobConfig] = None,
job_metadata: Optional[JobMetadata] = None,
) -> ImggJob:
- config = get_config()
imgg_config = get_config().cogt.imgg_config
job_metadata = job_metadata or JobMetadata(
- top_job_id=f"IMGGJob for {config.project_name}",
job_category=JobCategory.IMGG_JOB,
)
job_params = imgg_job_params or imgg_config.make_default_imgg_job_params()
@@ -44,10 +42,8 @@ class ImggJobFactory:
imgg_job_config: Optional[ImggJobConfig] = None,
job_metadata: Optional[JobMetadata] = None,
) -> ImggJob:
- config = get_config()
imgg_config = get_config().cogt.imgg_config
job_metadata = job_metadata or JobMetadata(
- top_job_id=f"IMGGJob for {config.project_name}",
job_category=JobCategory.IMGG_JOB,
)
imgg_prompt = ImggPrompt(positive_text=positive_text)
diff --git a/pipelex/cogt/llm/llm_job_factory.py b/pipelex/cogt/llm/llm_job_factory.py
index f96a99865..34c268140 100644
--- a/pipelex/cogt/llm/llm_job_factory.py
+++ b/pipelex/cogt/llm/llm_job_factory.py
@@ -31,7 +31,6 @@ class LLMJobFactory:
)
else:
job_metadata = JobMetadata(
- top_job_id=f"LLMJob for {config.project_name}",
job_category=JobCategory.LLM_JOB,
)
job_params = llm_job_params
@@ -106,7 +105,6 @@ class LLMJobFactory:
)
else:
job_metadata = JobMetadata(
- top_job_id=f"LLMJob for {config.project_name}",
job_category=JobCategory.LLM_JOB,
)
llm_prompt = LLMPrompt(
diff --git a/pipelex/cogt/ocr/ocr_job_factory.py b/pipelex/cogt/ocr/ocr_job_factory.py
index 9220dd05a..2633b0d0a 100644
--- a/pipelex/cogt/ocr/ocr_job_factory.py
+++ b/pipelex/cogt/ocr/ocr_job_factory.py
@@ -3,7 +3,6 @@ from typing import Optional
from pipelex.cogt.ocr.ocr_input import OcrInput
from pipelex.cogt.ocr.ocr_job import OcrJob
from pipelex.cogt.ocr.ocr_job_components import OcrJobConfig, OcrJobParams, OcrJobReport
-from pipelex.config import get_config
from pipelex.pipeline.job_metadata import JobCategory, JobMetadata
@@ -16,11 +15,9 @@ class OcrJobFactory:
ocr_job_config: Optional[OcrJobConfig] = None,
job_metadata: Optional[JobMetadata] = None,
) -> OcrJob:
- config = get_config()
- # TODO: manahge the param default sthrough the config
+ # TODO: manage the param default through the config
# ocr_config = get_config().cogt.ocr_config
job_metadata = job_metadata or JobMetadata(
- top_job_id=f"OCRJob for {config.project_name}",
job_category=JobCategory.OCR_JOB,
)
job_params = ocr_job_params or OcrJobParams.make_default_ocr_job_params()
diff --git a/pipelex/core/pipe_output.py b/pipelex/core/pipe_output.py
index 2b5810291..3be2b90b2 100644
--- a/pipelex/core/pipe_output.py
+++ b/pipelex/core/pipe_output.py
@@ -14,10 +14,12 @@ from pipelex.core.stuff_content import (
TextContent,
)
from pipelex.core.working_memory import WorkingMemory
+from pipelex.pipeline.pipeline_models import SpecialPipelineId
class PipeOutput(BaseModel):
working_memory: WorkingMemory = Field(default_factory=WorkingMemory)
+ pipeline_run_id: str = Field(default=SpecialPipelineId.UNTITLED)
@property
def main_stuff(self) -> Stuff:
diff --git a/pipelex/pipe_controllers/pipe_batch.py b/pipelex/pipe_controllers/pipe_batch.py
index 39f847d3b..c299f4d7b 100644
--- a/pipelex/pipe_controllers/pipe_batch.py
+++ b/pipelex/pipe_controllers/pipe_batch.py
@@ -163,4 +163,5 @@ class PipeBatch(PipeController):
return PipeOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
diff --git a/pipelex/pipe_controllers/pipe_parallel.py b/pipelex/pipe_controllers/pipe_parallel.py
index 773e80a49..081221bae 100644
--- a/pipelex/pipe_controllers/pipe_parallel.py
+++ b/pipelex/pipe_controllers/pipe_parallel.py
@@ -102,4 +102,5 @@ class PipeParallel(PipeController):
)
return PipeOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
diff --git a/pipelex/pipe_controllers/pipe_sequence.py b/pipelex/pipe_controllers/pipe_sequence.py
index 30f76eb36..ae53dc94b 100644
--- a/pipelex/pipe_controllers/pipe_sequence.py
+++ b/pipelex/pipe_controllers/pipe_sequence.py
@@ -51,4 +51,5 @@ class PipeSequence(PipeController):
return PipeOutput(
working_memory=current_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
diff --git a/pipelex/pipe_operators/pipe_func.py b/pipelex/pipe_operators/pipe_func.py
index e539398b3..b41e01292 100644
--- a/pipelex/pipe_operators/pipe_func.py
+++ b/pipelex/pipe_operators/pipe_func.py
@@ -59,5 +59,6 @@ class PipeFunc(PipeOperator):
pipe_output = PipeFuncOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
return pipe_output
diff --git a/pipelex/pipe_operators/pipe_img_gen.py b/pipelex/pipe_operators/pipe_img_gen.py
index 26216fa78..76de14f48 100644
--- a/pipelex/pipe_operators/pipe_img_gen.py
+++ b/pipelex/pipe_operators/pipe_img_gen.py
@@ -223,7 +223,6 @@ class PipeImgGen(PipeOperator):
nb_images=nb_images,
imgg_job_params=imgg_job_params,
imgg_job_config=imgg_config.imgg_job_config,
- wfid="imgg",
)
image_content_items: List[StuffContent] = []
for generated_image in generated_image_list:
@@ -247,7 +246,6 @@ class PipeImgGen(PipeOperator):
),
imgg_job_params=imgg_job_params,
imgg_job_config=imgg_config.imgg_job_config,
- wfid="imgg",
)
generated_image_url = generated_image.url
@@ -272,6 +270,7 @@ class PipeImgGen(PipeOperator):
pipe_output = PipeImgGenOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
return pipe_output
diff --git a/pipelex/pipe_operators/pipe_jinja2.py b/pipelex/pipe_operators/pipe_jinja2.py
index 9dc3c5517..ea8443d33 100644
--- a/pipelex/pipe_operators/pipe_jinja2.py
+++ b/pipelex/pipe_operators/pipe_jinja2.py
@@ -127,6 +127,7 @@ class PipeJinja2(PipeOperator):
pipe_output = PipeJinja2Output(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
return pipe_output
diff --git a/pipelex/pipe_operators/pipe_llm.py b/pipelex/pipe_operators/pipe_llm.py
index d4d46552d..4879e0ccc 100644
--- a/pipelex/pipe_operators/pipe_llm.py
+++ b/pipelex/pipe_operators/pipe_llm.py
@@ -311,7 +311,6 @@ class PipeLLM(PipeOperator):
job_metadata=job_metadata,
llm_prompt_for_text=llm_prompt_1,
llm_setting_main=self.llm_setting_main,
- wfid=f"{self.class_name}_gen_text",
)
the_content = TextContent(
@@ -395,6 +394,7 @@ class PipeLLM(PipeOperator):
pipe_output = PipeLLMOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
return pipe_output
@@ -433,7 +433,6 @@ class PipeLLM(PipeOperator):
llm_prompt_factory_for_object_list=llm_prompt_2_factory,
llm_setting_for_object_list=self.llm_setting_for_object_list,
nb_items=fixed_nb_output,
- wfid=task_desc,
)
else:
# We're generating a list of objects directly
@@ -445,7 +444,6 @@ class PipeLLM(PipeOperator):
llm_prompt_for_object_list=llm_prompt_1,
llm_setting_for_object_list=self.llm_setting_for_object_list_direct,
nb_items=fixed_nb_output,
- wfid=task_desc,
)
the_content = ListContent(items=generated_objects)
@@ -464,7 +462,6 @@ class PipeLLM(PipeOperator):
llm_setting_main=self.llm_setting_main,
llm_prompt_factory_for_object=llm_prompt_2_factory,
llm_setting_for_object=self.llm_setting_for_object,
- wfid=task_desc,
)
else:
# We're generating a single object directly
@@ -475,7 +472,6 @@ class PipeLLM(PipeOperator):
object_class=content_class,
llm_prompt_for_object=llm_prompt_1,
llm_setting_for_object=self.llm_setting_for_object_direct,
- wfid=task_desc,
)
the_content = generated_object
diff --git a/pipelex/pipe_operators/pipe_llm_prompt.py b/pipelex/pipe_operators/pipe_llm_prompt.py
index e7d107878..32ef5d6b9 100644
--- a/pipelex/pipe_operators/pipe_llm_prompt.py
+++ b/pipelex/pipe_operators/pipe_llm_prompt.py
@@ -208,6 +208,7 @@ class PipeLLMPrompt(PipeOperator):
pipe_output = PipeLLMPromptOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
return pipe_output
diff --git a/pipelex/pipe_operators/pipe_ocr.py b/pipelex/pipe_operators/pipe_ocr.py
index 744f499c7..956199b85 100644
--- a/pipelex/pipe_operators/pipe_ocr.py
+++ b/pipelex/pipe_operators/pipe_ocr.py
@@ -217,6 +217,7 @@ class PipeOcr(PipeOperator):
pipe_output = PipeOcrOutput(
working_memory=working_memory,
+ pipeline_run_id=job_metadata.pipeline_run_id,
)
return pipe_output
diff --git a/pipelex/pipe_works/pipe_router.py b/pipelex/pipe_works/pipe_router.py
index 5961b3c9e..d4339ed77 100644
--- a/pipelex/pipe_works/pipe_router.py
+++ b/pipelex/pipe_works/pipe_router.py
@@ -18,7 +18,6 @@ class PipeRouter(PipeRouterProtocol):
async def run_pipe_job(
self,
pipe_job: PipeJob,
- wfid: Optional[str] = None,
) -> PipeOutputType: # pyright: ignore[reportInvalidTypeVarUse]
log.debug(f"Start run_pipe_job: pipe_code={pipe_job.pipe.code}")
working_memory = pipe_job.working_memory
@@ -43,7 +42,6 @@ class PipeRouter(PipeRouterProtocol):
job_metadata: Optional[JobMetadata] = None,
working_memory: Optional[WorkingMemory] = None,
output_name: Optional[str] = None,
- wfid: Optional[str] = None,
) -> PipeOutputType: # pyright: ignore[reportInvalidTypeVarUse]
log.debug(f"Start run_pipe_code: pipe_code={pipe_code}")
pipe = get_required_pipe(pipe_code)
@@ -56,7 +54,6 @@ class PipeRouter(PipeRouterProtocol):
)
pipe_output: PipeOutputType = await self.run_pipe_job(
pipe_job=pipe_job,
- wfid=wfid,
)
log.debug(f"Completed run_pipe_code: pipe_code={pipe_code}")
return pipe_output
diff --git a/pipelex/pipe_works/pipe_router_protocol.py b/pipelex/pipe_works/pipe_router_protocol.py
index bc9c4ea90..ee014f4ee 100644
--- a/pipelex/pipe_works/pipe_router_protocol.py
+++ b/pipelex/pipe_works/pipe_router_protocol.py
@@ -11,7 +11,6 @@ class PipeRouterProtocol(Protocol):
async def run_pipe_job(
self,
pipe_job: PipeJob,
- wfid: Optional[str] = None,
) -> PipeOutputType: ... # pyright: ignore[reportInvalidTypeVarUse]
async def run_pipe_code(
@@ -21,5 +20,4 @@ class PipeRouterProtocol(Protocol):
job_metadata: Optional[JobMetadata] = None,
working_memory: Optional[WorkingMemory] = None,
output_name: Optional[str] = None,
- wfid: Optional[str] = None,
) -> PipeOutputType: ... # pyright: ignore[reportInvalidTypeVarUse]
diff --git a/pipelex/pipeline/execute.py b/pipelex/pipeline/execute.py
index b83b9dfd0..bb562359b 100644
--- a/pipelex/pipeline/execute.py
+++ b/pipelex/pipeline/execute.py
@@ -1,4 +1,4 @@
-from typing import Optional, Tuple
+from typing import Optional
from pipelex import pretty_print
from pipelex.core.pipe_output import PipeOutput
@@ -18,7 +18,7 @@ async def execute_pipeline(
output_multiplicity: Optional[PipeOutputMultiplicity] = None,
dynamic_output_concept_code: Optional[str] = None,
pipe_run_mode: Optional[PipeRunMode] = None,
-) -> Tuple[PipeOutput, str]:
+) -> PipeOutput:
"""Execute a pipeline and wait for its completion.
This function executes a pipe and returns its output along with the pipeline run ID.
@@ -81,4 +81,4 @@ async def execute_pipeline(
output_name=output_name,
)
- return await get_pipe_router().run_pipe_job(pipe_job), pipeline_run_id
+ return await get_pipe_router().run_pipe_job(pipe_job)
diff --git a/pipelex/pipeline/job_metadata.py b/pipelex/pipeline/job_metadata.py
index 104d60a2b..6ce2810dc 100644
--- a/pipelex/pipeline/job_metadata.py
+++ b/pipelex/pipeline/job_metadata.py
@@ -1,7 +1,7 @@
from datetime import datetime
from typing import List, Optional
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, Field
from pipelex.pipeline.pipeline_models import SpecialPipelineId
from pipelex.types import StrEnum
@@ -24,14 +24,13 @@ class UnitJobId(StrEnum):
class JobMetadata(BaseModel):
- model_config = ConfigDict(arbitrary_types_allowed=True)
-
+ job_name: Optional[str] = None
pipeline_run_id: str = Field(default=SpecialPipelineId.UNTITLED)
- top_job_id: Optional[str] = None
pipe_job_ids: Optional[List[str]] = None
+
content_generation_job_id: Optional[str] = None
- job_category: Optional[JobCategory] = None
unit_job_id: Optional[str] = None
+ job_category: Optional[JobCategory] = None
started_at: Optional[datetime] = Field(default_factory=lambda: datetime.now())
completed_at: Optional[datetime] = None
@@ -45,8 +44,6 @@ class JobMetadata(BaseModel):
def update(self, updated_metadata: "JobMetadata"):
if updated_metadata.job_category:
self.job_category = updated_metadata.job_category
- if updated_metadata.top_job_id:
- self.top_job_id = updated_metadata.top_job_id
if updated_metadata.pipe_job_ids:
self.pipe_job_ids = self.pipe_job_ids or []
self.pipe_job_ids.extend(updated_metadata.pipe_job_ids)
diff --git a/pipelex/pipeline/start.py b/pipelex/pipeline/start.py
index ac4b72d87..4fcc0b7e8 100644
--- a/pipelex/pipeline/start.py
+++ b/pipelex/pipeline/start.py
@@ -1,5 +1,5 @@
import asyncio
-from typing import Optional, Tuple
+from typing import Optional
from pipelex import pretty_print
from pipelex.core.pipe_output import PipeOutput
@@ -18,7 +18,7 @@ async def start_pipeline(
output_multiplicity: Optional[PipeOutputMultiplicity] = None,
dynamic_output_concept_code: Optional[str] = None,
pipe_run_mode: PipeRunMode = PipeRunMode.LIVE,
-) -> Tuple[str, asyncio.Task[PipeOutput]]:
+) -> asyncio.Task[PipeOutput]:
"""Start a pipeline in the background.
This function mirrors *execute_pipeline* but returns immediately with the
@@ -77,4 +77,4 @@ async def start_pipeline(
# Launch execution without awaiting the result.
task: asyncio.Task[PipeOutput] = asyncio.create_task(get_pipe_router().run_pipe_job(pipe_job))
- return pipeline_run_id, task
+ return task
diff --git a/tests/integration/pipelex/cogt/test_content_generator.py b/tests/integration/pipelex/cogt/test_content_generator.py
index 57c2b884e..3a2e75623 100644
--- a/tests/integration/pipelex/cogt/test_content_generator.py
+++ b/tests/integration/pipelex/cogt/test_content_generator.py
@@ -1,4 +1,4 @@
-from typing import List, cast
+from typing import List
import pytest
from pytest import FixtureRequest
@@ -64,9 +64,7 @@ class TestContentGenerator:
llm_setting_main = get_llm_deck().get_llm_setting(llm_setting_or_preset_id="llm_for_testing_gen_text")
text: str = await get_content_generator().make_llm_text(
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
llm_prompt_for_text=LLMPrompt(user_text=USER_TEXT_FOR_BASE),
llm_setting_main=llm_setting_main,
)
@@ -80,9 +78,7 @@ class TestContentGenerator:
llm_setting_for_object = get_llm_deck().get_llm_setting(llm_setting_or_preset_id="llm_for_testing_gen_object")
person_direct: Employee = await get_content_generator().make_object_direct(
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
object_class=Employee,
llm_prompt_for_object=LLMPrompt(user_text=USER_TEXT_FOR_SINGLE_PERSON),
llm_setting_for_object=llm_setting_for_object,
@@ -97,9 +93,7 @@ class TestContentGenerator:
llm_setting_for_object = get_llm_deck().get_llm_setting(llm_setting_or_preset_id="llm_for_testing_gen_object")
person_list_direct: List[Employee] = await get_content_generator().make_object_list_direct(
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
object_class=Employee,
llm_prompt_for_object_list=LLMPrompt(user_text=USER_TEXTS_FOR_PEOPLE_STR),
llm_setting_for_object_list=llm_setting_for_object,
@@ -113,9 +107,7 @@ class TestContentGenerator:
@pytest.mark.inference
async def test_make_image(self, request: FixtureRequest):
image: GeneratedImage = await get_content_generator().make_single_image(
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
imgg_handle=ImggHandle.SDXL_LIGHTNING,
imgg_prompt=ImggPrompt(
positive_text="A dog with sunglasses coding on a laptop",
@@ -141,9 +133,7 @@ class TestContentGenerator:
@pytest.mark.inference
async def test_make_ocr_extract_pages(self, request: FixtureRequest):
ocr_output = await get_content_generator().make_ocr_extract_pages(
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
ocr_handle=OcrHandle.MISTRAL_OCR,
ocr_input=OcrInput(image_uri=ImageTestCases.IMAGE_FILE_PATH_PNG),
ocr_job_params=OcrJobParams.make_default_ocr_job_params(),
@@ -159,9 +149,7 @@ class TestContentGenerator:
llm_setting_main = LLMSetting(llm_handle=BAD_HANDLE_TO_TEST_FAILURE, temperature=0.5, max_tokens=100)
with pytest.raises(LLMHandleNotFoundError) as excinfo:
await get_content_generator().make_llm_text(
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # pyright: ignore[reportUnknownMemberType]
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
llm_prompt_for_text=LLMPrompt(user_text=USER_TEXT_FOR_BASE),
llm_setting_main=llm_setting_main,
)
diff --git a/tests/integration/pipelex/pipes/test_image_inputs.py b/tests/integration/pipelex/pipes/test_image_inputs.py
index a958e4e0d..17f434a2c 100644
--- a/tests/integration/pipelex/pipes/test_image_inputs.py
+++ b/tests/integration/pipelex/pipes/test_image_inputs.py
@@ -1,5 +1,3 @@
-from typing import cast
-
import pytest
from pytest import FixtureRequest
@@ -35,9 +33,7 @@ class TestImageInputs:
pipe_code="extract_article_from_image",
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
working_memory=working_memory,
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
# Log output and generate report
@@ -74,9 +70,7 @@ class TestImageInputs:
pipe_code="describe_page",
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
working_memory=working_memory,
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
# Log output and generate report
diff --git a/tests/integration/pipelex/pipes/test_pipe_running_variants.py b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
index 967692317..722765cc7 100644
--- a/tests/integration/pipelex/pipes/test_pipe_running_variants.py
+++ b/tests/integration/pipelex/pipes/test_pipe_running_variants.py
@@ -1,4 +1,4 @@
-from typing import Any, Optional, Tuple, Type, cast
+from typing import Any, Optional, Tuple, Type
import pytest
from pytest import FixtureRequest
@@ -40,9 +40,7 @@ class TestPipeRunningVariants:
pipe_code=pipe_code,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
working_memory=working_memory,
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
log.verbose(pipe_output, title="pipe_output")
pretty_print(pipe_output, title="pipe_output")
@@ -70,9 +68,7 @@ class TestPipeRunningVariants:
pipe_code=pipe_code,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
working_memory=working_memory,
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
get_report_delegate().generate_report()
@@ -95,9 +91,7 @@ class TestPipeRunningVariants:
pipe_code=pipe_code,
pipe_run_params=PipeRunParamsFactory.make_run_params(pipe_run_mode=pipe_run_mode),
working_memory=WorkingMemory(),
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
get_report_delegate().generate_report()
@@ -129,9 +123,7 @@ class TestPipeRunningVariants:
output_multiplicity=output_multiplicity,
),
working_memory=WorkingMemory(),
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
get_report_delegate().generate_report()
@@ -167,9 +159,7 @@ class TestPipeRunningVariants:
pipe_run_mode=pipe_run_mode,
),
working_memory=working_memory,
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
pretty_print(pipe_output, title=f"run pipe '{pipe_code}'")
get_report_delegate().generate_report()
@@ -194,9 +184,7 @@ class TestPipeRunningVariants:
pipe_stack_limit=6,
pipe_run_mode=pipe_run_mode,
),
- job_metadata=JobMetadata(
- top_job_id=cast(str, request.node.originalname), # type: ignore
- ),
+ job_metadata=JobMetadata(job_name=request.node.originalname), # type: ignore
)
pretty_print(exc.value, title="exception")
assert expected_error_message in str(exc.value)
diff --git a/tests/unit/pipelex/client/test_api_serialization.py b/tests/unit/pipelex/client/test_api_serialization.py
index 91abfceff..5299c0843 100644
--- a/tests/unit/pipelex/client/test_api_serialization.py
+++ b/tests/unit/pipelex/client/test_api_serialization.py
@@ -3,8 +3,6 @@ from datetime import datetime
from decimal import Decimal
from enum import Enum
from pathlib import Path
-
-# Import additional types for complex testing
from typing import Any, Dict, List, Optional, cast
import pytest