Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
feature/Trace-event-ttl · L-260927-292588 (#1426)
Louis Choquel623c00955caf
diff --git a/.drift/acks/config-docs.toml b/.drift/acks/config-docs.tomlindex 21314f84d..8953fc986 100644--- a/.drift/acks/config-docs.toml+++ b/.drift/acks/config-docs.toml@@ -1,8 +1,8 @@contract = "config-docs"-digest = "sha256:66bf51a2e90466528e0059c0722fccebd1accb16236e9b81af538eb24a1b00ef"+digest = "sha256:2af32cbadd596a5408a68ca05db6073d687342949b12c224c77ae66318097426"reviewed_by = "Louis Choquel"-reviewed_at = "2026-10-11T20:09:33Z"-rationale = "clean-pass: ConfigLoader.find_project_root now stats each marker instead of asking Path.exists(), which from Python 3.14 swallows every OSError; the markers, the walk and the home stop are unchanged, and a directory that cannot be traversed raises as it already did on 3.13. No config model, default or documented behaviour changed, so docs/configuration and the migration ledgers stand."+reviewed_at = "2026-10-12T21:51:11Z"+rationale = "clean-pass: DynamoDBTracingConfig gains retention_days, an int of at least 1 with its default of 30 in pipelex/pipelex.toml and in the kit and project copies. The change is additive and its bound is an annotation the fingerprint records, so the regenerated head golden accounts for it and no ledger entry is needed. docs/configuration has no tracing page to go stale; the setting is documented with the tracing backends in docs/under-the-hood/execution-graph-tracing.md."[trigger_files]"pipelex/cogt/config_cogt.py" = "blob:293ef22ae123266e6a1886d4ffb07f455e5be9e3"@@ -10,14 +10,14 @@ rationale = "clean-pass: ConfigLoader.find_project_root now stats each marker in"pipelex/graph/graph_config.py" = "blob:dc0939a6d2730c0a10b88aa1a6e0b2dd583462a6""pipelex/language/mthds_config.py" = "blob:353009ad8c59dc5266bdd9dfa42b1ad18b67dca7""pipelex/methods/methods_config.py" = "blob:76ca9a0e64f8c10ef6a355aafd0d79d36b09f06a"-"pipelex/pipelex.toml" = "blob:95eed7d7186c0a6b25349893c002c16fc82b4d4d"+"pipelex/pipelex.toml" = "blob:ec186eff52f9a612964ec661cd56bf5b198a698b""pipelex/system/configuration/__init__.py" = "blob:8b137891791fe96927ad78e64b0aad7bded08bdc""pipelex/system/configuration/config_check.py" = "blob:2d5de29aba3d2ce205d3d82b4623f37b53108332""pipelex/system/configuration/config_loader.py" = "blob:8703fc3e5a82d7c6a3d148ea195c25044c2f0fe0""pipelex/system/configuration/config_model.py" = "blob:b7d141d559c0302f0d8c56859ae0a8edeb9bfdf0""pipelex/system/configuration/config_root.py" = "blob:c7d43851270912d17413cfc6a511247d3c7cc160""pipelex/system/configuration/config_surface.py" = "blob:49424a8103a5611f5b2885135dd699273bde57c8"-"pipelex/system/configuration/configs.py" = "blob:4a95dd4fe7d17ad5b32c1b61063dbba43b18e40b"+"pipelex/system/configuration/configs.py" = "blob:fb032dd59182f0c188201567026dbab0cc8f8774""pipelex/system/configuration/pipe_func_config.py" = "blob:dc8890e914fbcdb611147e42eaa0cd87b32288a5""pipelex/system/telemetry/telemetry_config.py" = "blob:068a785d97a2d52ee43e82e723cd944b8b6601bc""pipelex/tools/aws/aws_config.py" = "blob:0a7c9dbf276e61355b1fa79b9db8caf252450a39"diff --git a/.pipelex/pipelex.toml b/.pipelex/pipelex.tomlindex 60ff964f1..665b27ceb 100644--- a/.pipelex/pipelex.toml+++ b/.pipelex/pipelex.toml@@ -162,6 +162,9 @@ traces_dir = ".pipelex/traces"[runtime.tracing.dynamodb]table_name = ""region = ""+# Days a trace event is kept: every item is written with a `ttl` (epoch seconds) this many days past its write,+# which the table must use as its TTL attribute for DynamoDB to delete expired events.+retention_days = 30##################################################################################################### Pluginsdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex fd1b4d891..89e50d6d6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,11 @@# Changelog+## [Unreleased]++### Fixed++- **Trace events written to DynamoDB expire (Breaking)**: the DynamoDB event log now writes every event with a `ttl` in epoch seconds, its write time plus the new `runtime.tracing.dynamodb.retention_days` (30 by default, at least 1), so a table that expires items on `ttl` deletes a run's events, full output data included, where it used to keep them forever. Only events written from this version on carry a `ttl`, so the rows an earlier version wrote stay until they are deleted or given a `ttl` by hand. `DynamoDBEventLog` takes `retention_days` and now takes all its arguments by keyword; `make_event_log` reads the setting from the configuration.+## [v0.85.1] - 2026-10-12### Fixeddiff --git a/docs/under-the-hood/execution-graph-tracing.md b/docs/under-the-hood/execution-graph-tracing.mdindex 8ef637df3..ceaba87e8 100644--- a/docs/under-the-hood/execution-graph-tracing.md+++ b/docs/under-the-hood/execution-graph-tracing.md@@ -271,6 +271,8 @@ graph_spec = manager.close_tracer(pipeline_run_id)Trace events travel through an `EventLogProtocol` backend (`pipelex/tracing/`): the tracer emits events into it during the run (write side, wired in `pipeline_run_setup`), and `assemble_tracing` reads them back after the run to build the `GraphSpec` and usage aggregates (read side, triggered from `PipeRun.run`). Both sides normally build their backend instance independently from `tracing_config` via `make_event_log` — NDJSON files or DynamoDB bridge the two instances through external storage.+The DynamoDB backend (`pipelex/tracing/dynamodb_event_log.py`) writes each event as one item keyed by the run (`PK = PIPELINE_RUN#{pipeline_run_id}`, `SK = EVENT#{workflow_id}#{writer_id}#{sequence}`), and every item carries a `ttl` in epoch seconds, set to the time of the write plus `runtime.tracing.dynamodb.retention_days` (30 days by default). DynamoDB deletes an item once its `ttl` has passed, provided the table names `ttl` as its TTL attribute, so a run's events, which by default hold its full output data, are not kept forever. The retention counts from the write rather than from the event's own `timestamp`, and a Temporal replay that re-emits an event overwrites the same item, pushing its expiry back by no more than the time between the two writes. A reader that needs a run's events, such as the graph and usage assembly, must therefore read them within the retention. Only the items written by a version that sets `ttl` expire: an item an earlier version wrote has no `ttl`, so it stays in the table until it is deleted or given one.+The assembled usage rides back on `pipe_output.tokens_usages` (with any assembly failure on `usage_assembly_error`), which the sync `/execute` response returns directly. For delivery-enabled runs (a storage target set), the delivery executor (`pipelex/pipe_run/delivery_executor.py`) also persists it as a `tokens_usages.json` result artifact — `{"tokens_usages": [...], "usage_assembly_error": null}` — next to `working_memory.json`, the `main_stuff.*` renders, and the graph outputs (`graphspec.json` with its three companions `pipe_io_contracts.json`, `input_form.json` and `output_form.json`, see [Outputs](#outputs)), so a durable client polling result files gets the same usage records a sync caller does. The artifact is written unconditionally on every successful result delivery: explicit nulls mean usage assembly was off for that run. A failed run stores no result files at all, so an absent `tokens_usages.json` means either the run failed or it was delivered before the artifact existed — tell those apart from the delivery status, not from the file's presence.For fully in-process runs, `pipelex.runtime_hub.scoped_event_log` pins one shared instance for both sides instead:@@ -436,6 +438,25 @@ ReactFlow HTML is generated directly from GraphSpec — no intermediate ViewSpec## Configuration+### Event Log Backend++```toml+# pipelex.toml (default values)+[runtime.tracing]+is_enabled = true+backend = "ndjson" # "ndjson" or "dynamodb"++[runtime.tracing.ndjson]+traces_dir = ".pipelex/traces"++[runtime.tracing.dynamodb]+table_name = "" # Required when backend = "dynamodb"+region = "" # Required when backend = "dynamodb"+retention_days = 30 # Each event's `ttl` is its write time plus this many days (at least 1)+```++See [Event-Log Transport and the Scoped Override](#event-log-transport-and-the-scoped-override) for what each backend does with these settings.+### GraphConfig```tomldiff --git a/pipelex/kit/configs/pipelex.toml b/pipelex/kit/configs/pipelex.tomlindex 60ff964f1..665b27ceb 100644--- a/pipelex/kit/configs/pipelex.toml+++ b/pipelex/kit/configs/pipelex.toml@@ -162,6 +162,9 @@ traces_dir = ".pipelex/traces"[runtime.tracing.dynamodb]table_name = ""region = ""+# Days a trace event is kept: every item is written with a `ttl` (epoch seconds) this many days past its write,+# which the table must use as its TTL attribute for DynamoDB to delete expired events.+retention_days = 30##################################################################################################### Pluginsdiff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index 95eed7d71..ec186eff5 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -174,6 +174,9 @@ traces_dir = ".pipelex/traces"[runtime.tracing.dynamodb]table_name = ""region = ""+# Days a trace event is kept: every item is written with a `ttl` (epoch seconds) this many days past its write,+# which the table must use as its TTL attribute for DynamoDB to delete expired events.+retention_days = 30[runtime.observer]observer_dir = "results/observer"diff --git a/pipelex/migration/goldens/pipelex-config/[email protected] b/pipelex/migration/goldens/pipelex-config/[email protected]index a89e69188..b0b973792 100644--- a/pipelex/migration/goldens/pipelex-config/[email protected]+++ b/pipelex/migration/goldens/pipelex-config/[email protected]@@ -1488,6 +1488,15 @@"default": "","open_node": false},+ "runtime.tracing.dynamodb.retention_days": {+ "value_type": "int",+ "required": true,+ "default": 30,+ "open_node": false,+ "constraints": {+ "ge": 1+ }+ },"runtime.tracing.dynamodb.table_name": {"value_type": "str","required": true,diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.tomlindex 95eed7d71..ec186eff5 100644--- a/pipelex/pipelex.toml+++ b/pipelex/pipelex.toml@@ -174,6 +174,9 @@ traces_dir = ".pipelex/traces"[runtime.tracing.dynamodb]table_name = ""region = ""+# Days a trace event is kept: every item is written with a `ttl` (epoch seconds) this many days past its write,+# which the table must use as its TTL attribute for DynamoDB to delete expired events.+retention_days = 30[runtime.observer]observer_dir = "results/observer"diff --git a/pipelex/system/configuration/configs.py b/pipelex/system/configuration/configs.pyindex 4a95dd4fe..fb032dd59 100644--- a/pipelex/system/configuration/configs.py+++ b/pipelex/system/configuration/configs.py@@ -81,6 +81,9 @@ class NdjsonTracingConfig(ConfigModel):class DynamoDBTracingConfig(ConfigModel):table_name: strregion: str+ # How long a trace event is kept: each item is written with a `ttl` this many days past its write, the attribute+ # the events table expires items on.+ retention_days: Annotated[int, Field(ge=1)]class TracingConfig(ConfigModel):diff --git a/pipelex/tracing/dynamodb_event_log.py b/pipelex/tracing/dynamodb_event_log.pyindex 5ee578217..d07ccc5b7 100644--- a/pipelex/tracing/dynamodb_event_log.py+++ b/pipelex/tracing/dynamodb_event_log.py@@ -4,6 +4,7 @@ Stores trace events in a DynamoDB table with the schema:PK: PIPELINE_RUN#{pipeline_run_id}SK: EVENT#{workflow_id}#{writer_id}#{sequence:010d}payload: dict (full event via model_dump)+ ttl: epoch seconds at which DynamoDB may delete the item, the write time plus the configured retentionLegacy emissions write writer_id="primary" so existing rows remain valid.@@ -12,6 +13,7 @@ Requires: pip install "pipelex[dynamodb]""""import threading+import timefrom typing import Anyfrom pydantic import TypeAdapter, ValidationError@@ -34,6 +36,8 @@ except ImportError:_any_trace_event_adapter: TypeAdapter[TraceEvent] = TypeAdapter(AnyTraceEvent)+_SECONDS_PER_DAY = 86_400+class DynamoDBEventLog(EventLogProtocol):"""Event log backed by AWS DynamoDB.@@ -42,22 +46,31 @@ class DynamoDBEventLog(EventLogProtocol):provides natural deduplication for Temporal replay re-emissions.Read path: Query on PK, returns events sorted by SK which encodes(workflow_id, sequence) — matching the NDJSON ordering contract.+ Retention: every item carries a `ttl` in epoch seconds, `retention_days`+ past its write, which is the attribute the events table expires items on,+ so a run's events are deleted by DynamoDB instead of being kept forever.WARNING: Do not call emit() from inside a Temporal workflow body — thesynchronous boto3 HTTP call will block the workflow thread and triggerthe deadlock detector. Use BufferingEventLog + act_flush_trace_events instead."""- def __init__(self, table_name: str, region: str, writer_id: str = "primary") -> None:+ def __init__(self, *, table_name: str, region: str, retention_days: int, writer_id: str = "primary") -> None:if boto3 is None:lib_name = "boto3"lib_extra_name = "dynamodb"msg = "boto3 is required for the DynamoDB event log backend."raise MissingDependencyError(lib_name, lib_extra_name, msg)+ # The config refuses a retention under one day too, but a direct construction bypasses it,+ # and such a retention would expire every event as it is written.+ if retention_days < 1:+ msg = f"The DynamoDB event log needs a retention of at least one day, got retention_days={retention_days}."+ raise EventLogSetupError(msg)self._table_name = table_nameself._region = regionself._writer_id = writer_id+ self._retention_seconds = retention_days * _SECONDS_PER_DAYself._sequence: int = 0self._sequence_lock = threading.Lock()# Translate construction-time botocore failures (e.g. a misconfigured region) into our@@ -107,6 +120,9 @@ class DynamoDBEventLog(EventLogProtocol):@overridedef emit(self, event: TraceEvent) -> None:"""Write a single event to DynamoDB. Synchronous and idempotent."""+ # The retention counts from the write rather than from the event's own timestamp, which is for display only.+ # A replay's re-emission overwrites the item under the same key and only pushes its expiry a little later.+ ttl = int(time.time()) + self._retention_secondsitem: dict[str, Any] = {"PK": self._make_pk(event.pipeline_run_id),"SK": self._make_sk(event.workflow_id, writer_id=event.writer_id, sequence=event.sequence),@@ -115,6 +131,7 @@ class DynamoDBEventLog(EventLogProtocol):"writer_id": event.writer_id,"sequence": event.sequence,"payload": event.model_dump_json(),+ "ttl": ttl,}self._table.put_item(Item=item)@@ -199,7 +216,7 @@ class DynamoDBEventLog(EventLogProtocol):@overridedef cleanup(self, pipeline_run_id: str) -> None:- """Delete all events for a pipeline run. In practice TTL handles this."""+ """Delete all events for a pipeline run, ahead of the expiry their `ttl` sets."""response = self._table.query(KeyConditionExpression=self._key_condition(pipeline_run_id),ProjectionExpression="PK, SK",diff --git a/pipelex/tracing/event_log_factory.py b/pipelex/tracing/event_log_factory.pyindex a713555b5..1f0913250 100644--- a/pipelex/tracing/event_log_factory.py+++ b/pipelex/tracing/event_log_factory.py@@ -27,5 +27,6 @@ def make_event_log(tracing_config: TracingConfig, *, writer_id: str = "primary")return DynamoDBEventLog(table_name=tracing_config.dynamodb.table_name,region=tracing_config.dynamodb.region,+ retention_days=tracing_config.dynamodb.retention_days,writer_id=writer_id,)diff --git a/tests/unit/pipelex/tracing/test_dynamodb_event_log.py b/tests/unit/pipelex/tracing/test_dynamodb_event_log.pyindex f505f6212..c266ce8d5 100644--- a/tests/unit/pipelex/tracing/test_dynamodb_event_log.py+++ b/tests/unit/pipelex/tracing/test_dynamodb_event_log.py@@ -1,21 +1,80 @@+import timefrom typing import Anyimport pytestfrom botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError+from pydantic import ValidationErrorfrom pytest_mock import MockerFixture+from pipelex.system.configuration.configs import DynamoDBTracingConfig, TracingConfigfrom pipelex.tracing.dynamodb_event_log import DynamoDBEventLog+from pipelex.tracing.event_log_factory import make_event_log+from pipelex.tracing.event_log_protocol import EventLogProtocolfrom pipelex.tracing.exceptions import EventLogReadError, EventLogSetupError+from tests.unit.pipelex.tracing.conftest import make_trace_event+_SECONDS_PER_DAY = 86_400-class TestDynamoDBEventLogReadEvents:- def _make_event_log_with_table(self, mocker: MockerFixture) -> tuple[DynamoDBEventLog, Any]:++class TestDynamoDBEventLog:+ def _make_event_log_with_table(self, mocker: MockerFixture, *, retention_days: int = 30) -> tuple[DynamoDBEventLog, Any]:# boto3.resource(...) does not perform any network call at construction.- event_log = DynamoDBEventLog(table_name="trace-events-test", region="us-east-1")+ event_log = DynamoDBEventLog(table_name="trace-events-test", region="us-east-1", retention_days=retention_days)mock_table = mocker.MagicMock()mocker.patch.object(event_log, "_table", new=mock_table)return event_log, mock_table+ @staticmethod+ def _emit_and_capture_ttl(event_log: EventLogProtocol, mock_table: Any) -> tuple[int, int, int]:+ """Emit one event and return the written item's ttl with the epoch seconds read just before and just after."""+ before = int(time.time())+ event_log.emit(make_trace_event())+ after = int(time.time())+ item: dict[str, Any] = mock_table.put_item.call_args.kwargs["Item"]+ ttl = item["ttl"]+ assert isinstance(ttl, int)+ return ttl, before, after++ @pytest.mark.parametrize("retention_days", [1, 30, 365])+ def test_emit_writes_ttl_retention_days_past_the_write(self, mocker: MockerFixture, retention_days: int) -> None:+ """Every item carries a `ttl` in epoch seconds, the attribute the events table expires items on."""+ event_log, mock_table = self._make_event_log_with_table(mocker, retention_days=retention_days)++ ttl, before, after = self._emit_and_capture_ttl(event_log, mock_table)++ retention_seconds = retention_days * _SECONDS_PER_DAY+ assert before + retention_seconds <= ttl <= after + retention_seconds++ def test_make_event_log_writes_the_configured_retention(self, mocker: MockerFixture) -> None:+ """The factory hands `runtime.tracing.dynamodb.retention_days` to the writer, so the setting decides the `ttl`."""+ tracing_config = TracingConfig.model_validate(+ {+ "is_enabled": True,+ "backend": "dynamodb",+ "dynamodb": {"table_name": "trace-events-test", "region": "us-east-1", "retention_days": 7},+ }+ )+ event_log = make_event_log(tracing_config)+ assert isinstance(event_log, DynamoDBEventLog)+ mock_table = mocker.MagicMock()+ mocker.patch.object(event_log, "_table", new=mock_table)++ ttl, before, after = self._emit_and_capture_ttl(event_log, mock_table)++ assert before + 7 * _SECONDS_PER_DAY <= ttl <= after + 7 * _SECONDS_PER_DAY++ @pytest.mark.parametrize("retention_days", [0, -1])+ def test_config_refuses_a_retention_under_one_day(self, retention_days: int) -> None:+ """A retention of zero or less would expire every event as it is written, so the configuration refuses it."""+ with pytest.raises(ValidationError):+ DynamoDBTracingConfig.model_validate({"table_name": "trace-events-test", "region": "us-east-1", "retention_days": retention_days})++ @pytest.mark.parametrize("retention_days", [0, -1])+ def test_init_refuses_a_retention_under_one_day(self, retention_days: int) -> None:+ """A direct construction bypasses the config's check, so the writer refuses such a retention itself."""+ with pytest.raises(EventLogSetupError, match="at least one day"):+ DynamoDBEventLog(table_name="trace-events-test", region="us-east-1", retention_days=retention_days)+@pytest.mark.parametrize("backend_error",[@@ -57,6 +116,6 @@ class TestDynamoDBEventLogReadEvents:mocker.patch("pipelex.tracing.dynamodb_event_log.boto3.resource", side_effect=construction_error)with pytest.raises(EventLogSetupError) as exc_info:- DynamoDBEventLog(table_name="trace-events-test", region="bad-region")+ DynamoDBEventLog(table_name="trace-events-test", region="bad-region", retention_days=30)assert exc_info.value.__cause__ is construction_errordiff --git a/tests/unit/pipelex/tracing/test_event_log_sequence_threadsafe.py b/tests/unit/pipelex/tracing/test_event_log_sequence_threadsafe.pyindex a69abc2ce..076d8e5ec 100644--- a/tests/unit/pipelex/tracing/test_event_log_sequence_threadsafe.py+++ b/tests/unit/pipelex/tracing/test_event_log_sequence_threadsafe.py@@ -59,7 +59,7 @@ class TestEventLogSequenceThreadSafety:event_log = BufferingEventLog()case "dynamodb":mocker.patch("pipelex.tracing.dynamodb_event_log.boto3")- event_log = DynamoDBEventLog(table_name="t", region="us-east-1")+ event_log = DynamoDBEventLog(table_name="t", region="us-east-1", retention_days=30)case _:msg = f"Unknown backend_name: {backend_name}"raise ValueError(msg)diff --git a/tests/unit/pipelex/tracing/test_writer_id_schema.py b/tests/unit/pipelex/tracing/test_writer_id_schema.pyindex 07d587481..7ef62145b 100644--- a/tests/unit/pipelex/tracing/test_writer_id_schema.py+++ b/tests/unit/pipelex/tracing/test_writer_id_schema.py@@ -200,7 +200,7 @@ class TestWriterIdSchema:return_value=_StubResource(),)- event_log = DynamoDBEventLog(table_name="my_table", region="us-east-1", writer_id="act_pid42")+ event_log = DynamoDBEventLog(table_name="my_table", region="us-east-1", retention_days=30, writer_id="act_pid42")event_log.emit(_make_pipe_start(sequence=0, writer_id="act_pid42"))assert len(captured_items) == 1