Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Guard NdjsonEventLog file-handle cache with a dedicated lock

Louis Choquel1925e8bc6285
diff --git a/pipelex/tracing/ndjson_event_log.py b/pipelex/tracing/ndjson_event_log.py
index 07155820b..ca72d52cf 100644
--- a/pipelex/tracing/ndjson_event_log.py
+++ b/pipelex/tracing/ndjson_event_log.py
@@ -46,6 +46,7 @@ class NdjsonEventLog(EventLogProtocol):
self._file_handles: dict[tuple[str, str, str], IO[str]] = {}
self._sequence: int = 0
self._sequence_lock = threading.Lock()
+ self._handles_lock = threading.Lock()
self._writer_id = writer_id
@property
@@ -96,11 +97,14 @@ class NdjsonEventLog(EventLogProtocol):
handle = self._file_handles.get(cache_key)
if handle is None:
- run_dir = os.path.join(self._traces_dir, event.pipeline_run_id)
- os.makedirs(run_dir, exist_ok=True)
- file_path = os.path.join(run_dir, self._file_name_for(event.workflow_id, event.writer_id))
- handle = open(file_path, "a", encoding="utf-8") # noqa: SIM115
- self._file_handles[cache_key] = handle
+ with self._handles_lock:
+ handle = self._file_handles.get(cache_key)
+ if handle is None:
+ run_dir = os.path.join(self._traces_dir, event.pipeline_run_id)
+ os.makedirs(run_dir, exist_ok=True)
+ file_path = os.path.join(run_dir, self._file_name_for(event.workflow_id, event.writer_id))
+ handle = open(file_path, "a", encoding="utf-8") # noqa: SIM115
+ self._file_handles[cache_key] = handle
handle.write(event.model_dump_json() + "\n")
handle.flush()