Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

fix(checkpointer): unify conditional pending cleanup

xfgong32e5ca4862a3
diff --git a/cubeloop/checkpointer/base.py b/cubeloop/checkpointer/base.py
index aa2fa38..ea4f88f 100644
--- a/cubeloop/checkpointer/base.py
+++ b/cubeloop/checkpointer/base.py
@@ -44,6 +44,20 @@ class Checkpointer(Protocol):
"""
...
+ async def clear_pending_request_if_matches(
+ self,
+ thread_id: str,
+ *,
+ question_id: str,
+ run_id: str,
+ ) -> bool:
+ """Atomically clear pending state when its question and run still match.
+
+ Returns True when the matching pending request was cleared and False
+ when the thread has no pending request or either identifier changed.
+ """
+ ...
+
async def snapshot(self, thread_id: str, *, after_run_id: str) -> list[Message]:
"""Return messages of completed runs of `thread_id` up through
and including `after_run_id`, in source seq order. Raises
diff --git a/cubeloop/checkpointer/memory.py b/cubeloop/checkpointer/memory.py
index fba6fc5..6d5b236 100644
--- a/cubeloop/checkpointer/memory.py
+++ b/cubeloop/checkpointer/memory.py
@@ -85,6 +85,25 @@ class MemoryCheckpointer:
async def load_pending_request(self, thread_id: str) -> HitlRequest | None:
return self._pending.get(thread_id)
+ async def clear_pending_request_if_matches(
+ self,
+ thread_id: str,
+ *,
+ question_id: str,
+ run_id: str,
+ ) -> bool:
+ async with self._lock:
+ request = self._pending.get(thread_id)
+ if (
+ request is None
+ or request.question_id != question_id
+ or self._pending_run_id.get(thread_id) != run_id
+ ):
+ return False
+ self._pending.pop(thread_id, None)
+ self._pending_run_id.pop(thread_id, None)
+ return True
+
async def load_pending_run_id(self, thread_id: str) -> str | None:
return self._pending_run_id.get(thread_id)
diff --git a/cubeloop/checkpointer/mysql/checkpointer.py b/cubeloop/checkpointer/mysql/checkpointer.py
index 9ea5017..027cbad 100644
--- a/cubeloop/checkpointer/mysql/checkpointer.py
+++ b/cubeloop/checkpointer/mysql/checkpointer.py
@@ -740,6 +740,28 @@ class MySQLCheckpointer:
return HitlRequest.model_validate_json(raw)
return HitlRequest.model_validate(raw)
+ async def clear_pending_request_if_matches(
+ self,
+ thread_id: str,
+ *,
+ question_id: str,
+ run_id: str,
+ ) -> bool:
+ """Atomically clear pending state when its question and run still match."""
+ assert self._pool is not None
+ async with self._pool.acquire() as conn:
+ async with conn.cursor() as cur:
+ await cur.execute(
+ "UPDATE cubepi_threads "
+ "SET pending_request = NULL, run_id = NULL, "
+ "updated_at = CURRENT_TIMESTAMP "
+ "WHERE thread_id = %s "
+ "AND JSON_UNQUOTE(JSON_EXTRACT(pending_request, '$.question_id')) = %s "
+ "AND run_id = %s",
+ (thread_id, question_id, run_id),
+ )
+ return cur.rowcount == 1
+
async def load_pending_run_id(self, thread_id: str) -> str | None:
"""Return the run_id of the currently pending HITL request.
diff --git a/cubeloop/checkpointer/sqlite.py b/cubeloop/checkpointer/sqlite.py
index 6f6f4a9..349c466 100644
--- a/cubeloop/checkpointer/sqlite.py
+++ b/cubeloop/checkpointer/sqlite.py
@@ -326,6 +326,23 @@ class SQLiteCheckpointer:
row = await cursor.fetchone()
return HitlRequest.model_validate_json(row[0]) if row else None
+ async def clear_pending_request_if_matches(
+ self,
+ thread_id: str,
+ *,
+ question_id: str,
+ run_id: str,
+ ) -> bool:
+ assert self._db is not None
+ async with self._lock, _writer_txn(self._db):
+ cursor = await self._db.execute(
+ "DELETE FROM thread_pending_request "
+ "WHERE thread_id = ? AND json_extract(request_json, '$.question_id') = ? "
+ "AND run_id = ?",
+ (thread_id, question_id, run_id),
+ )
+ return cursor.rowcount == 1
+
async def load_pending_run_id(self, thread_id: str) -> str | None:
assert self._db is not None
async with self._lock:
diff --git a/tests/checkpointer/test_mysql_pending_request.py b/tests/checkpointer/test_mysql_pending_request.py
index dd7f054..bcca5b0 100644
--- a/tests/checkpointer/test_mysql_pending_request.py
+++ b/tests/checkpointer/test_mysql_pending_request.py
@@ -36,6 +36,26 @@ async def test_mysql_clear_pending_request(clean_mysql_db) -> None:
assert loaded is None
+async def test_mysql_compare_and_clear_pending_request(clean_mysql_db) -> None:
+ await _setup_schema_v2(clean_mysql_db)
+ async with MySQLCheckpointer(clean_mysql_db) as cp:
+ await cp.save_pending_request("t-1", _req(qid="q-current"), run_id="run-new")
+
+ assert not await cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-replaced", run_id="run-new"
+ )
+ assert not await cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-current", run_id="run-old"
+ )
+ assert await cp.load_pending("t-1") == (_req(qid="q-current"), "run-new")
+
+ assert await cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-current", run_id="run-new"
+ )
+ assert await cp.load_pending("t-1") is None
+
+
@pytest.mark.asyncio
async def test_mysql_pending_request_creates_thread_row_lazily(clean_mysql_db) -> None:
"""save_pending_request must upsert the thread row so calling it on an unknown
diff --git a/tests/hitl/test_checkpointer_pending_request.py b/tests/hitl/test_checkpointer_pending_request.py
index 38216cd..6d475e1 100644
--- a/tests/hitl/test_checkpointer_pending_request.py
+++ b/tests/hitl/test_checkpointer_pending_request.py
@@ -39,6 +39,24 @@ async def test_memory_clear_pending():
assert await cp.load_pending_request("t-1") is None
+async def test_memory_compare_and_clear_pending():
+ cp = MemoryCheckpointer()
+ await cp.save_pending_request("t-1", _req(qid="q-current"), run_id="run-new")
+
+ assert not await cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-replaced", run_id="run-new"
+ )
+ assert not await cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-current", run_id="run-old"
+ )
+ assert await cp.load_pending("t-1") == (_req(qid="q-current"), "run-new")
+
+ assert await cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-current", run_id="run-new"
+ )
+ assert await cp.load_pending("t-1") is None
+
+
async def test_sqlite_save_and_load_pending(sqlite_cp):
assert await sqlite_cp.load_pending_request("t-1") is None
req = _req()
@@ -53,6 +71,26 @@ async def test_sqlite_clear_pending(sqlite_cp):
assert await sqlite_cp.load_pending_request("t-1") is None
+async def test_sqlite_compare_and_clear_pending(sqlite_cp):
+ await sqlite_cp.save_pending_request("t-1", _req(qid="q-current"), run_id="run-new")
+
+ assert not await sqlite_cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-replaced", run_id="run-new"
+ )
+ assert not await sqlite_cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-current", run_id="run-old"
+ )
+ assert await sqlite_cp.load_pending("t-1") == (
+ _req(qid="q-current"),
+ "run-new",
+ )
+
+ assert await sqlite_cp.clear_pending_request_if_matches(
+ "t-1", question_id="q-current", run_id="run-new"
+ )
+ assert await sqlite_cp.load_pending("t-1") is None
+
+
async def test_sqlite_create_table_idempotent(sqlite_cp):
"""Re-opening a checkpointer DB with existing pending_request table is safe."""
await sqlite_cp.save_pending_request("t-1", _req())
diff --git a/website/docs/guides/checkpointing/custom.md b/website/docs/guides/checkpointing/custom.md
index c994e29..69a98c9 100644
--- a/website/docs/guides/checkpointing/custom.md
+++ b/website/docs/guides/checkpointing/custom.md
@@ -5,7 +5,7 @@ description: "Implement a custom checkpointer backend for CubeLoop using the Che
# Custom Checkpointing Backends
-The `Checkpointer` protocol is three async methods:
+The core history portion of the `Checkpointer` protocol is three async methods:
```python
class Checkpointer(Protocol):
@@ -14,8 +14,15 @@ class Checkpointer(Protocol):
async def save_extra(self, thread_id: str, extra: dict[str, Any]) -> None: ...
```
-That's the whole contract. Implement it for Redis, DynamoDB, S3, a
-filesystem, an in-memory dict — anything that can append-and-list.
+That is enough for basic message and extra persistence. The full protocol also
+contains run, fork, and HITL lifecycle methods; implement those when the host
+uses the corresponding features. In particular,
+`clear_pending_request_if_matches(thread_id, question_id=..., run_id=...)`
+must compare and clear atomically so stale resume cleanup cannot erase a newer
+pending request. See the API reference for the complete signatures.
+
+You can implement the protocol for Redis, DynamoDB, S3, a filesystem, an
+in-memory dict — anything that can append-and-list.
## When the agent calls each method
diff --git a/website/docs/guides/checkpointing/mysql.md b/website/docs/guides/checkpointing/mysql.md
index cc64412..44f34eb 100644
--- a/website/docs/guides/checkpointing/mysql.md
+++ b/website/docs/guides/checkpointing/mysql.md
@@ -193,6 +193,14 @@ under a row lock and writes the merged dict, rather than using
`JSON_MERGE_PATCH`, whose null-deletion and deep-merge semantics differ
from `dict.update`.)
+## Conditional pending cleanup
+
+Hosts that coordinate HITL resumes across workers can use
+`clear_pending_request_if_matches(thread_id, question_id=..., run_id=...)`. It
+clears the pending request and owning run ID only if both stored identifiers
+still match. The comparison and clear are one MySQL update, so stale cleanup
+cannot erase a newer request, even if another run reuses the question ID.
+
## Forks
`MySQLCheckpointer` implements the v4 `snapshot` / `fork` /
diff --git a/website/docs/guides/checkpointing/postgres.md b/website/docs/guides/checkpointing/postgres.md
index 106437f..f1aac8a 100644
--- a/website/docs/guides/checkpointing/postgres.md
+++ b/website/docs/guides/checkpointing/postgres.md
@@ -193,7 +193,8 @@ Hosts that coordinate HITL resumes across workers can use
clears the pending request and owning run ID only if both stored identifiers
still match. The comparison and clear are one PostgreSQL update, so a newer
request written by another worker is never erased by stale cleanup, even if it
-reuses the same question ID in a different run.
+reuses the same question ID in a different run. The same API is available on
+all built-in checkpointers.
## Forks
diff --git a/website/docs/guides/checkpointing/sqlite.md b/website/docs/guides/checkpointing/sqlite.md
index 2903734..5d8b024 100644
--- a/website/docs/guides/checkpointing/sqlite.md
+++ b/website/docs/guides/checkpointing/sqlite.md
@@ -154,6 +154,13 @@ have multiple processes writing the same `agent.db`:
If you need multi-process writers on shared threads, jump to
[Postgres](./postgres) which uses an advisory lock per thread.
+## Conditional pending cleanup
+
+`clear_pending_request_if_matches(thread_id, question_id=..., run_id=...)`
+deletes pending state only when both stored identifiers still match. SQLite
+performs the comparison and delete in one statement, while the checkpointer's
+writer lock serializes it with other writes in the same process.
+
## Forks
`SQLiteCheckpointer` implements the v4 `snapshot` / `fork` /
diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/custom.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/custom.md
index a7bfb40..467011f 100644
--- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/custom.md
+++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/custom.md
@@ -5,7 +5,7 @@ description: "使用 Checkpointer protocol 为 CubeLoop 实现自定义 checkpoi
# 自定义 Checkpointing 后端
-`Checkpointer` protocol 只有三个 async 方法:
+`Checkpointer` protocol 的核心历史记录部分包含三个 async 方法:
```python
class Checkpointer(Protocol):
@@ -14,7 +14,13 @@ class Checkpointer(Protocol):
async def save_extra(self, thread_id: str, extra: dict[str, Any]) -> None: ...
```
-这就是全部契约。你可以为 Redis、DynamoDB、S3、文件系统、内存字典实现它——任何支持追加和列表操作的存储均可。
+这足以支持基本的消息和 extra 持久化。完整 protocol 还包括 run、fork 和
+HITL 生命周期方法;宿主使用相应功能时也需要实现这些方法。其中
+`clear_pending_request_if_matches(thread_id, question_id=..., run_id=...)`
+必须以原子方式完成比较和清理,避免过期的恢复流程误删新的 pending request。
+完整签名请查看 API 参考。
+
+你可以为 Redis、DynamoDB、S3、文件系统、内存字典实现该 protocol——任何支持追加和列表操作的存储均可。
## agent 何时调用各方法
diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/mysql.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/mysql.md
index e13e551..5f2a856 100644
--- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/mysql.md
+++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/mysql.md
@@ -149,6 +149,14 @@ cubepi_schema_version
`save_extra` 执行浅层顶级合并,而非替换——与 Postgres 和 SQLite 的行为相同。先写入 `{"foo": 1}` 再写入 `{"bar": 2}` 后,结果为 `{"foo": 1, "bar": 2}`。(内部实现是在行锁保护下读取当前 `extra`,然后写入合并后的字典,而非使用 `JSON_MERGE_PATCH`——后者的 null 删除和深度合并语义与 `dict.update` 不同。)
+## 条件式清理 pending request
+
+跨 worker 协调 HITL 恢复的宿主可以调用
+`clear_pending_request_if_matches(thread_id, question_id=..., run_id=...)`。
+只有当前 pending request 的问题 ID 和所属 run ID 都仍然匹配时,它才会
+清除这两个字段。比较与清理在一条 MySQL 更新中完成,因此即使另一个 run
+复用了相同问题 ID,旧 worker 也不会误删新请求。
+
## Forks
`MySQLCheckpointer` 实现了 v4 的 `snapshot` / `fork` /
diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/postgres.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/postgres.md
index 8164cc5..e029a84 100644
--- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/postgres.md
+++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/postgres.md
@@ -173,6 +173,7 @@ extra = cubepi_threads.extra || EXCLUDED.extra
只有数据库中当前 pending request 的问题 ID 和所属 run ID 都仍然匹配时,
它才会清除这两个字段。比较与清理在一条 PostgreSQL 更新中完成,因此即使
另一个 worker 的新 run 复用了相同问题 ID,旧 worker 也不会误删新请求。
+所有内置 checkpointer 都提供相同的 API。
## Fork
diff --git a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/sqlite.md b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/sqlite.md
index f678cdb..6990932 100644
--- a/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/sqlite.md
+++ b/website/i18n/zh-Hans/docusaurus-plugin-content-docs/current/guides/checkpointing/sqlite.md
@@ -127,6 +127,13 @@ Checkpointer 内部对每次读写都用 `asyncio.Lock`。SQLite 本身允许
如果你需要跨进程共享 thread 写入,请用 [Postgres](./postgres),
它对每个 thread 都用 advisory lock。
+## 条件式清理 pending request
+
+`clear_pending_request_if_matches(thread_id, question_id=..., run_id=...)`
+只有在当前 pending request 的问题 ID 和所属 run ID 都仍然匹配时,才会
+删除 pending 状态。SQLite 在一条语句中完成比较和删除;checkpointer 的
+写锁会将它与同一进程内的其他写入串行化。
+
## 文件放哪里
生产环境用绝对路径: