The Memory Layer for AI Agents - Drop-in memory infrastructure for AI agents and apps. Context that persists. Built for production.
Python agentic-memoryagentic-memory-systemagentsaiai-agentschatgptgenaillmlong-term-memorymemorymemory-managementpythonragstate-management
fix(plugins): stop delivering telemetry events twice, and stop losing parked ones (#7324)
Saket Aryan33629990956c
diff --git a/integrations/agent-plugin-core/python/telemetry.py b/integrations/agent-plugin-core/python/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/agent-plugin-core/python/telemetry.py+++ b/integrations/agent-plugin-core/python/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int:diff --git a/integrations/agent-plugin-core/tests/test_spool_delivery.py b/integrations/agent-plugin-core/tests/test_spool_delivery.pynew file mode 100644index 000000000..accd631a6--- /dev/null+++ b/integrations/agent-plugin-core/tests/test_spool_delivery.py@@ -0,0 +1,446 @@+"""Delivery semantics of the telemetry spool: no duplicates, no starvation.++These run against a built host's core in-process (not a subprocess) because they+need to inject failures into ``_post``. The identity tests next door cover the+uninitialised-process case that needs a real interpreter.+"""++from __future__ import annotations++import importlib+import json+import os+import sys+import time+from pathlib import Path++import pytest++CORE_ROOT = Path(__file__).resolve().parents[1]+REPOSITORY_ROOT = CORE_ROOT.parents[1]+HOST_CORE = REPOSITORY_ROOT / "integrations" / "claude-code-plugin" / "core"++pytestmark = pytest.mark.skipif(not HOST_CORE.exists(), reason="claude-code-plugin is not built")+++def telemetry(tmp_path, monkeypatch):+ # CI runs this directory and claude-code-plugin/tests in ONE pytest process,+ # and that suite's conftest sets MEM0_TELEMETRY=false at import, process-wide.+ # Without this the whole file silently no-ops: record() returns early and+ # every assertion sees an empty spool. Do not rely on ambient env.+ monkeypatch.setenv("MEM0_TELEMETRY", "true")+ monkeypatch.setenv("MEM0_CODE_DATA_DIR", str(tmp_path / "data"))+ monkeypatch.syspath_prepend(str(HOST_CORE))++ # Save and RESTORE rather than delete. claude-code-plugin/tests/conftest.py+ # imports memory_core once at collection and calls configure_harness() on it;+ # dropping the module left a later re-import with default harness config, so+ # tests in that suite failed depending on collection order.+ names = ("telemetry", "memory_core", "_harness_id")+ saved = {name: sys.modules.get(name) for name in names}+ for name in names:+ sys.modules.pop(name, None)++ module = importlib.import_module("telemetry")+ monkeypatch.setattr(module, "resolve_distinct_id", lambda: ("[email protected]", ""))+ try:+ yield module+ finally:+ for name in names:+ sys.modules.pop(name, None)+ if saved[name] is not None:+ sys.modules[name] = saved[name]+++def _delivered(payloads):+ return [event for payload in payloads if "batch" in payload for event in payload["batch"]]+++def test_a_partial_failure_does_not_redeliver_what_already_arrived(telemetry):+ """Defect 2a: flush kept the whole claim on failure and retried from the top.++ 150 events across two batches, the second failing, previously delivered 250.+ """+ for index in range(150):+ telemetry.record("search", index=index)++ sent: list[dict] = []+ calls = {"n": 0}++ def flaky(payload, url):+ calls["n"] += 1+ if calls["n"] == 2: # second batch fails+ return False+ sent.append(payload)+ return True++ telemetry._post = flaky+ telemetry.flush()++ telemetry._post = lambda payload, url: sent.append(payload) or True+ telemetry.flush()++ events = _delivered(sent)+ assert len(events) == 150+ assert len({event["uuid"] for event in events}) == 150+++def test_a_fresh_claim_is_not_immediately_stealable(telemetry):+ """Defect 2b: rename preserves mtime, so a claim inherited the spool's age.++ With the last write older than the stale threshold, a claim made now looked+ abandoned the instant it existed and a second sender took it over.+ """+ telemetry.record("search")+ spool = telemetry._spool_path()+ old = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(spool, (old, old))++ first = telemetry._claim_spool()+ assert first is not None++ # A second sender starting right now must find nothing to take.+ assert telemetry._claim_parked(first.parent) is None+++def test_a_live_final_attempt_is_not_deleted_by_another_sender(telemetry):+ """Review finding: exhaustion was judged before liveness, so owners lost batches.++ Claiming a parked file bumps its attempt count and refreshes its mtime. Once+ the count reaches the budget, the owner draining it looked exhausted to every+ other sender, which unlinked the file out from under it. Everything in that+ batch was gone, which is precisely the loss this PR exists to stop.+ """+ telemetry.record("search", reason="owned-by-the-first-sender")+ spool = telemetry._spool_path()+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(spool, (stale, stale))++ claim = telemetry._claim_spool()+ assert claim is not None++ # Walk it to the final attempt, ageing it each round so it can be re-claimed.+ # _claim_spool hands back a0 and _release_claim keeps the name, so it takes+ # one full round per attempt to reach the budget.+ for _ in range(telemetry.MAX_CLAIM_ATTEMPTS):+ # Carry the marker through each rewrite so the final assertion proves the+ # events survived, not merely that some file with the right name did.+ telemetry._release_claim(claim, [{"event": "code.search", "uuid": "owned-by-the-first-sender"}])+ parked = sorted(claim.parent.glob("telemetry-*.sending"))+ assert parked, "the batch was dropped while still inside its budget"+ os.utime(parked[0], (stale, stale))+ claim = telemetry._claim_parked(claim.parent)+ assert claim is not None++ assert telemetry._claim_attempt(claim) >= telemetry.MAX_CLAIM_ATTEMPTS+ assert claim.exists()++ # The owner is draining it right now: fresh mtime, live lease.+ second_sender = telemetry._claim_parked(claim.parent)++ assert second_sender is None, "a second sender took a batch under a live lease"+ assert claim.exists(), "a second sender deleted a batch its owner was draining"+ assert "owned-by-the-first-sender" in claim.read_text(encoding="utf-8")+++def test_an_exhausted_batch_is_still_discarded_once_its_lease_lapses(telemetry):+ """The liveness check must defer the cleanup, not cancel it.++ Guards the obvious over-correction: skipping live claims is only safe if an+ abandoned one at the same attempt count is still reaped on a later run.+ """+ telemetry.record("search")+ spool = telemetry._spool_path()+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(spool, (stale, stale))++ claim = telemetry._claim_spool()+ assert claim is not None+ exhausted = claim.parent / telemetry._claim_name(telemetry.MAX_CLAIM_ATTEMPTS)+ claim.replace(exhausted)+ os.utime(exhausted, (stale, stale))++ assert telemetry._claim_parked(exhausted.parent) is None+ assert not exhausted.exists(), "an abandoned exhausted batch was left behind forever"+++def test_a_parked_batch_is_drained_behind_the_live_spool(telemetry):+ """Defect 6: parked claims were only reachable when no spool existed.++ Because sessions keep recording there usually was one, so a batch parked by+ a failed send waited until the 7-day expiry deleted it unsent — even though+ its own presence is what starts the sender.+ """+ telemetry.record("parked")+ telemetry._post = lambda payload, url: False+ telemetry.flush()++ parked = list(telemetry.memory_core.data_dir().glob("telemetry-*.sending"))+ assert len(parked) == 1+ old = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(parked[0], (old, old))++ telemetry.record("fresh")+ sent: list[dict] = []+ telemetry._post = lambda payload, url: sent.append(payload) or True+ telemetry.flush()++ names = {event["event"] for event in _delivered(sent)}+ assert names == {"code.parked", "code.fresh"}+++def test_a_batch_is_retried_until_the_budget_is_spent_not_discarded(telemetry):+ """Expiry discards what failed repeatedly, not what merely sat for a while.++ The budget is the attempt count, because age cannot be one: every re-claim+ touches the mtime and every release backdates it, so age never accumulates.+ """+ telemetry.record("parked")+ telemetry._post = lambda payload, url: False+ telemetry.flush()++ parked = list(telemetry.memory_core.data_dir().glob("telemetry-*.sending"))+ assert len(parked) == 1+ assert telemetry._claim_attempt(parked[0]) < telemetry.MAX_CLAIM_ATTEMPTS++ sent: list[dict] = []+ telemetry._post = lambda payload, url: sent.append(payload) or True+ telemetry.flush()++ assert [event["event"] for event in _delivered(sent)] == ["code.parked"]+++def test_progress_is_recorded_after_every_batch(telemetry):+ """A crash repeats at most one batch, not the whole file."""+ for index in range(250):+ telemetry.record("search", index=index)++ calls = {"n": 0}++ def die_after_two(payload, url):+ calls["n"] += 1+ if calls["n"] > 2:+ return False+ return True++ telemetry._post = die_after_two+ telemetry.flush()++ parked = list(telemetry.memory_core.data_dir().glob("telemetry-*.sending"))+ assert len(parked) == 1+ remaining = parked[0].read_text(encoding="utf-8").strip().splitlines()+ # Two batches of 100 landed; only the last 50 should still be pending.+ assert len(remaining) == 50+ assert json.loads(remaining[0])["properties"]["index"] == 200+++def test_the_heartbeat_actually_refreshes_the_lease(telemetry):+ """The claim rewrite doubles as the lease heartbeat.++ Previously asserted `SEND_TIMEOUT * 4 < CLAIM_STALE_SECONDS`, which compares+ two constants and executes none of the code under test. Drive the real+ rewrite and watch the mtime move instead.+ """+ for index in range(150):+ telemetry.record("search", index=index)+ claim = telemetry._claim_spool()+ assert claim is not None++ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(claim, (stale, stale))+ assert time.time() - claim.stat().st_mtime > telemetry.CLAIM_STALE_SECONDS++ telemetry._rewrite_claim(claim, [{"event": "code.x", "properties": {}}])+ assert time.time() - claim.stat().st_mtime < telemetry.CLAIM_STALE_SECONDS+++def test_an_undeliverable_batch_is_eventually_given_up_on(telemetry):+ """Expiry has to be reachable from a state the state machine can produce.++ It was not: every re-claim touched the mtime and every release backdated it+ by a fixed amount, so age hovered near the stale threshold and the 7-day+ expiry never fired. An undeliverable batch lived on disk forever, and+ spawn_flush saw it and started a sender on every hook.+ """+ telemetry.record("doomed")+ telemetry._post = lambda payload, url: False++ directory = telemetry.memory_core.data_dir()+ for _ in range(telemetry.MAX_CLAIM_ATTEMPTS + 3):+ telemetry.flush()+ # Attempts now carry a cooldown, so a released claim is not instantly+ # reclaimable. Age it to stand in for the wall time a real retry waits;+ # without this the loop spins inside one cooldown and proves nothing.+ for parked in directory.glob("telemetry-*.sending"):+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(parked, (stale, stale))++ leftover = list(directory.glob("telemetry-*.sending"))+ assert leftover == [], f"batch never given up on: {[p.name for p in leftover]}"+++def test_a_batch_that_cannot_be_read_is_not_counted_as_delivered(telemetry):+ """Review finding: a read failure reported 'everything delivered'.++ Nothing was posted, so calling it delivered lets flush() carry on to other+ claims as though this batch had arrived, and hides the failure from the one+ signal that says the run went badly. It also must not quarantine: a briefly+ unreadable file is retryable, and moving it to .corrupt discards the events+ over a transient filesystem error, because nothing ever re-globs .corrupt.+ """+ telemetry.record("search")+ spool = telemetry._spool_path()+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(spool, (stale, stale))+ claim = telemetry._claim_spool()+ assert claim is not None++ original = Path.read_text++ def unreadable(self, *args, **kwargs):+ if self == claim:+ raise OSError(5, "I/O error")+ return original(self, *args, **kwargs)++ Path.read_text = unreadable+ try:+ sent, delivered = telemetry._drain(claim)+ finally:+ Path.read_text = original++ assert sent == 0+ assert delivered is False, "an unread batch was reported as delivered"+ assert claim.exists(), "a transient read error discarded the batch"+ assert not list(claim.parent.glob("*.corrupt")), "quarantined over a transient error"+++def test_undecodable_content_is_still_quarantined_and_the_run_continues(telemetry):+ """The other half: genuinely unrecoverable content must not block the run.++ Guards the over-correction. If every read problem returned undelivered, one+ torn file would stop every later claim on every flush, forever.+ """+ telemetry.record("search")+ spool = telemetry._spool_path()+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(spool, (stale, stale))+ claim = telemetry._claim_spool()+ assert claim is not None+ claim.write_bytes(b"\xff\xfe torn \x00 write")++ sent, delivered = telemetry._drain(claim)++ assert (sent, delivered) == (0, True)+ assert not claim.exists()+ assert list(claim.parent.glob("*.corrupt")), "unrecoverable content was not quarantined"+++def test_retries_are_spread_over_real_time_not_burned_at_once(telemetry):+ """Review finding: releasing straight to reclaimable spent the budget instantly.++ Two senders hitting one momentary failure could walk a batch from attempt 0+ to the limit within seconds and discard it, when a retry a minute later would+ have delivered. Each release now has to age past a cooldown that grows with+ the attempts already spent.+ """+ telemetry.record("doomed")+ telemetry._post = lambda payload, url: False++ directory = telemetry.memory_core.data_dir()+ telemetry.flush()++ parked = list(directory.glob("telemetry-*.sending"))+ assert parked, "the batch was discarded on its first failure"+ assert telemetry._claim_attempt(parked[0]) == 0++ # Second sender, immediately: the cooldown has not elapsed, so it must not+ # be able to spend another attempt.+ telemetry.flush()+ still = list(directory.glob("telemetry-*.sending"))+ assert len(still) == 1+ assert telemetry._claim_attempt(still[0]) <= 1, "burned attempts without waiting"+++def test_a_legacy_claim_filename_is_not_mistaken_for_a_huge_attempt_count(telemetry):+ """The old shape is telemetry-<pid>-<hex>.sending, and hex can start with 'a'."""+ assert telemetry._claim_attempt(Path("telemetry-999-deadbeef.sending")) == 0+ assert telemetry._claim_attempt(Path("telemetry-999-a1234567.sending")) == 0+ assert telemetry._claim_attempt(Path("telemetry-999-deadbeef-a2.sending")) == 2+++def test_a_torn_claim_is_quarantined_not_deleted(telemetry):+ """A non-empty file that parses to nothing is the remainder, not garbage."""+ telemetry.record("search")+ claim = telemetry._claim_spool()+ claim.write_bytes(b"\xff\xfe not utf-8 at all")+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(claim, (stale, stale))++ sent = telemetry.flush()++ assert sent == 0+ assert not claim.exists()+ quarantined = list(telemetry.memory_core.data_dir().glob("*.corrupt"))+ assert len(quarantined) == 1, "torn claim was destroyed instead of kept"+++def test_a_failed_rewrite_stops_instead_of_redelivering(telemetry):+ """Ignoring the rewrite result reintroduced the duplicates this PR fixes."""+ for index in range(250):+ telemetry.record("search", index=index)++ telemetry._rewrite_claim = lambda claim, remaining: False+ delivered = []+ telemetry._post = lambda payload, url: delivered.extend(payload.get("batch", [])) or True++ telemetry.flush()+ assert len(delivered) == 100, f"kept going after a failed rewrite: {len(delivered)}"+++def test_partial_files_are_swept(telemetry):+ """Nothing else globs *.partial, so a crash mid-rename orphans one forever."""+ data_dir = telemetry.memory_core.data_dir()+ data_dir.mkdir(parents=True, exist_ok=True)+ debris = data_dir / "telemetry-1-abc-a0.1.partial"+ debris.write_text("x", encoding="utf-8")+ old = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(debris, (old, old))++ telemetry.flush()+ assert not debris.exists()+++def test_quarantined_batches_are_eventually_collected(telemetry):+ """Nothing re-globs .corrupt, so without a sweep they live on disk forever.++ Kept much longer than .partial debris on purpose: a quarantined batch is the+ only remaining evidence of events that could not be delivered.+ """+ directory = telemetry.memory_core.data_dir()+ directory.mkdir(parents=True, exist_ok=True)+ fresh = directory / "telemetry-1-aaaaaaaa-a0.corrupt"+ old = directory / "telemetry-2-bbbbbbbb-a0.corrupt"+ for path in (fresh, old):+ path.write_text("torn", encoding="utf-8")+ expired = time.time() - (telemetry.CLAIM_EXPIRY_SECONDS + 60)+ os.utime(old, (expired, expired))++ telemetry._sweep_debris(directory)++ assert fresh.exists(), "a recent quarantine was discarded before anyone could look at it"+ assert not old.exists(), "an expired quarantine was left on disk forever"+++def test_temp_files_orphaned_by_a_kill_are_collected(telemetry):+ """_write_identity and _install_salt unlink in a finally, which SIGKILL skips."""+ directory = telemetry.memory_core.data_dir()+ directory.mkdir(parents=True, exist_ok=True)+ orphan = directory / "telemetry-salt.999.tmp"+ orphan.write_text("abandoned", encoding="utf-8")+ stale = time.time() - (telemetry.CLAIM_STALE_SECONDS + 60)+ os.utime(orphan, (stale, stale))++ telemetry._sweep_debris(directory)++ assert not orphan.exists(), "a killed process left a temp file on disk forever"diff --git a/integrations/antigravity-plugin/core/telemetry.py b/integrations/antigravity-plugin/core/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/antigravity-plugin/core/telemetry.py+++ b/integrations/antigravity-plugin/core/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int:diff --git a/integrations/claude-code-plugin/core/telemetry.py b/integrations/claude-code-plugin/core/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/claude-code-plugin/core/telemetry.py+++ b/integrations/claude-code-plugin/core/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int:diff --git a/integrations/claude-code-plugin/tests/test_telemetry.py b/integrations/claude-code-plugin/tests/test_telemetry.pyindex 6029fa98a..de0db10c6 100644--- a/integrations/claude-code-plugin/tests/test_telemetry.py+++ b/integrations/claude-code-plugin/tests/test_telemetry.py@@ -199,9 +199,11 @@ def test_a_stale_claim_is_reclaimed(isolated_env, monkeypatch):telemetry.record("search")orphan = telemetry._claim_spool()assert orphan is not None- monkeypatch.setattr(- telemetry.time, "time", lambda: orphan.stat().st_mtime + telemetry.CLAIM_STALE_SECONDS + 1- )+ # Frozen rather than re-stat'd per call: flush() drains the live spool and+ # then looks for parked claims in the same run, so by the second look this+ # file no longer exists.+ stale_now = orphan.stat().st_mtime + telemetry.CLAIM_STALE_SECONDS + 1+ monkeypatch.setattr(telemetry.time, "time", lambda: stale_now)with patch.object(telemetry, "_post", lambda payload, url: True):assert telemetry.flush() == 1@@ -211,9 +213,13 @@ def test_an_expired_claim_is_dropped(isolated_env, monkeypatch):telemetry.record("search")orphan = telemetry._claim_spool()assert orphan is not None- monkeypatch.setattr(- telemetry.time, "time", lambda: orphan.stat().st_mtime + telemetry.CLAIM_EXPIRY_SECONDS + 1- )+ expired_now = orphan.stat().st_mtime + telemetry.CLAIM_EXPIRY_SECONDS + 1+ monkeypatch.setattr(telemetry.time, "time", lambda: expired_now)+ # Expiry now only discards a batch that was genuinely retried and failed,+ # so age alone is not enough — age it past the attempt budget too.+ retried = orphan.parent / orphan.name.replace("-a0.", f"-a{telemetry.MAX_CLAIM_ATTEMPTS}.")+ orphan.replace(retried)+ os.utime(retried, (expired_now, expired_now - telemetry.CLAIM_EXPIRY_SECONDS - 1))assert telemetry._claim_spool() is Noneassert not list(memory_core.data_dir().glob("telemetry-*.sending"))diff --git a/integrations/codex-plugin/core/telemetry.py b/integrations/codex-plugin/core/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/codex-plugin/core/telemetry.py+++ b/integrations/codex-plugin/core/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int:diff --git a/integrations/cursor-plugin/core/telemetry.py b/integrations/cursor-plugin/core/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/cursor-plugin/core/telemetry.py+++ b/integrations/cursor-plugin/core/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int:diff --git a/integrations/kimi-plugin/core/telemetry.py b/integrations/kimi-plugin/core/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/kimi-plugin/core/telemetry.py+++ b/integrations/kimi-plugin/core/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int:diff --git a/integrations/mem0-agent-plugin/core/telemetry.py b/integrations/mem0-agent-plugin/core/telemetry.pyindex 6d7aeecae..a4375f002 100644--- a/integrations/mem0-agent-plugin/core/telemetry.py+++ b/integrations/mem0-agent-plugin/core/telemetry.py@@ -100,6 +100,16 @@ BATCH_SIZE = 100SEND_TIMEOUT = 5CLAIM_STALE_SECONDS = 120CLAIM_EXPIRY_SECONDS = 7 * 24 * 60 * 60+# A batch is only discarded once it has genuinely been retried this many times.+MAX_CLAIM_ATTEMPTS = 3+# Parked claims drained per run, after the live spool. Bounded so a long backlog+# cannot turn one flush into an unbounded send loop.+MAX_PARKED_PER_RUN = 3+# Added to the wait before a released claim becomes reclaimable, per attempt+# already spent. Releasing straight to "reclaimable now" let two senders burn the+# whole budget within seconds of one another on a single momentary failure, and+# discard a batch a retry a minute later would have delivered.+RETRY_COOLDOWN_SECONDS = 60def is_enabled() -> bool:@@ -399,38 +409,201 @@ def spawn_flush() -> bool:return False+def _claim_name(attempt: int = 0) -> str:+ """Claim filename. The attempt count rides in the name so the 7-day expiry+ only ever discards a batch that was actually retried and failed."""+ return f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}-a{attempt}.sending"+++def _claim_attempt(claim: Path) -> int:+ """Attempts recorded in a claim filename; 0 for the pre-attempt-count shape.++ Anchored on field position, not on a leading "a": the legacy shape is+ ``telemetry-<pid>-<hex>.sending`` and a hex id such as ``a1234567`` would+ otherwise parse as attempt 1234567 and be discarded unsent on the first+ flush after an upgrade.+ """+ stem = claim.name[: -len(".sending")] if claim.name.endswith(".sending") else claim.name+ parts = stem.split("-")+ if len(parts) != 4:+ return 0+ tail = parts[3]+ if tail.startswith("a") and tail[1:].isdigit():+ return int(tail[1:])+ return 0+++def _touch(path: Path) -> None:+ """Refresh mtime so a claim's age measures time since it was claimed.++ ``Path.replace`` is ``os.rename``, which preserves mtime — so a claim created+ after a quiet minute inherited the spool's last-write time and looked+ abandoned the instant it was made. A second sender would then take it over+ while the first was still posting, and both would deliver the batch.+ """+ try:+ os.utime(path, None)+ except OSError:+ pass++def _claim_spool() -> Path | None:"""Rename the spool aside so exactly one sender owns each batch."""directory = memory_core.data_dir()- claim = directory / f"telemetry-{os.getpid()}-{uuid.uuid4().hex[:8]}.sending"+ claim = directory / _claim_name()spool = _spool_path()try:spool.replace(claim)+ _touch(claim)return claimexcept OSError:pass+ return _claim_parked(directory)+++def _sweep_debris(directory: Path) -> None:+ """Remove files nothing else will ever pick up again.++ *.partial is a temp file orphaned by a crash between write and rename.+ *.corrupt is a batch quarantined for undecodable content. No glob in this+ module matches either, so without this they accumulate on disk for the life+ of the install.++ Quarantined batches are kept far longer than debris: they are the only+ evidence left of events that could not be delivered, and someone diagnosing+ a report of missing telemetry has to be able to find one.+ """+ now = time.time()+ for debris in directory.glob("telemetry-*.partial"):+ try:+ if now - debris.stat().st_mtime > CLAIM_STALE_SECONDS:+ debris.unlink()+ except OSError:+ continue+ for quarantined in directory.glob("telemetry-*.corrupt"):+ try:+ if now - quarantined.stat().st_mtime > CLAIM_EXPIRY_SECONDS:+ quarantined.unlink()+ except OSError:+ continue+ # The same reasoning covers *.tmp. _write_identity and _install_salt both+ # create one and unlink it in a finally, which a SIGKILL skips, and no glob+ # in this module matches the leftovers either.+ for temporary in directory.glob("telemetry-*.tmp"):+ try:+ if now - temporary.stat().st_mtime > CLAIM_STALE_SECONDS:+ temporary.unlink()+ except OSError:+ continue+++def _claim_parked(directory: Path) -> Path | None:+ """Take the oldest abandoned claim, if any lease has actually expired.++ Kept separate from the live spool so flush() can drain both in one run.+ Previously parked batches were only reachable when no spool existed at all,+ and because sessions keep recording there usually was one — so a batch+ parked by a failed send waited until the 7-day expiry deleted it unsent,+ even though its own presence is what started the sender.+ """now = time.time()- for orphan in sorted(directory.glob("telemetry-*.sending")):+ for orphan in sorted(directory.glob("telemetry-*.sending"), key=_safe_mtime):try:age = now - orphan.stat().st_mtimeexcept OSError:continue- if age > CLAIM_EXPIRY_SECONDS:+ if age < CLAIM_STALE_SECONDS:+ # Someone else holds a live lease on it. This check has to come+ # first. Claiming a file bumps its attempt count and refreshes its+ # mtime, so a sender that has just taken the final attempt looks+ # exhausted to everyone else while it is actively draining. Judging+ # exhaustion before liveness let a second sender unlink a batch out+ # from under its owner, losing every event in it.+ continue+ # Attempts, not age. Every re-claim touches the mtime and every release+ # backdates it by a fixed amount, so age is pinned near the stale+ # threshold and never reaches the expiry. Age stays only as a backstop+ # for files that never carried an attempt marker.+ if _claim_attempt(orphan) >= MAX_CLAIM_ATTEMPTS or age > CLAIM_EXPIRY_SECONDS:try:orphan.unlink()except OSError:passcontinue- if age < CLAIM_STALE_SECONDS:- continue+ claim = orphan.parent / _claim_name(_claim_attempt(orphan) + 1)try:orphan.replace(claim)+ _touch(claim)return claimexcept OSError:continuereturn None+def _safe_mtime(path: Path) -> float:+ try:+ return path.stat().st_mtime+ except OSError:+ return 0.0+++def _rewrite_claim(claim: Path, remaining: list[dict[str, Any]]) -> bool:+ """Persist the unsent remainder, atomically, and refresh the lease.++ Called after every successful batch. Two jobs: a retry resumes where the+ send stopped instead of re-posting from the top, and the rewrite doubles as+ the lease heartbeat, so a slow sender does not have its claim stolen+ mid-flight. Interval is one batch, well inside CLAIM_STALE_SECONDS.+ """+ if not remaining:+ try:+ claim.unlink()+ except OSError:+ pass+ return True+ temporary = claim.with_suffix(f".{os.getpid()}.partial")+ try:+ payload = "".join(json.dumps(event, separators=(",", ":"), default=str) + "\n" for event in remaining)+ # fsync before the rename: without it the rename can land while the+ # bytes have not, and the claim comes back empty or truncated after a+ # crash. _drain then reads zero events and unlinks it.+ with open(temporary, "w", encoding="utf-8") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ temporary.replace(claim)+ _touch(claim)+ return True+ except OSError:+ try:+ temporary.unlink()+ except OSError:+ pass+ return False+++def _release_claim(claim: Path, remaining: list[dict[str, Any]]) -> None:+ """Persist the remainder and drop the lease, because this sender has given up.++ Distinct from the per-batch heartbeat: heartbeating on the way out would+ make an abandoned batch look actively owned for a further+ CLAIM_STALE_SECONDS, delaying the retry for no reason. Ageing it past the+ threshold lets the next flush pick it up immediately, while the attempt+ count in the filename still bounds how many times that can happen.+ """+ if not _rewrite_claim(claim, remaining):+ return+ try:+ # Backdate past the stale threshold so the next flush can pick it up,+ # minus a cooldown that grows with the attempts already spent. Clamped so+ # the mtime never lands in the future, which would read as a live lease.+ cooldown = min(_claim_attempt(claim) * RETRY_COOLDOWN_SECONDS, CLAIM_STALE_SECONDS)+ released = time.time() - CLAIM_STALE_SECONDS - 1 + cooldown+ os.utime(claim, (released, released))+ except OSError:+ pass++def _resolve_email(key: str) -> str:"""Trade the API key for the account email so events join other Mem0 surfaces."""url = os.environ.get("MEM0_API_URL", memory_core.DEFAULT_API_URL).rstrip("/") + "/v1/ping/"@@ -478,16 +651,61 @@ def resolve_distinct_id() -> tuple[str, str]:def flush() -> int:- """Drain claimed spools to PostHog and return the number of events sent."""+ """Drain the live spool, then any parked claims, and return events sent."""if not is_enabled():return 0- claim = _claim_spool()+ sent, delivered = _drain(_claim_spool())+ if not delivered:+ # The network is failing. Retrying other batches now would only burn+ # their attempt budget against the same broken connection.+ return sent++ # Parked batches used to starve behind the live spool indefinitely. Bounded+ # per run so a long backlog cannot turn one flush into an unbounded loop.+ directory = memory_core.data_dir()+ _sweep_debris(directory)+ for _ in range(MAX_PARKED_PER_RUN):+ parked = _claim_parked(directory)+ if parked is None:+ break+ count, delivered = _drain(parked)+ sent += count+ if not delivered:+ break+ return sent+++def _drain(claim: Path | None) -> tuple[int, bool]:+ """Post one claimed batch file, recording progress after every batch.++ Returns (events sent, whether everything was delivered).+ """if claim is None:- return 0+ return 0, Truetry:lines = claim.read_text(encoding="utf-8").splitlines()+ except ValueError:+ # UnicodeDecodeError from a torn write: the content is unrecoverable, so+ # quarantine rather than retry. flush() runs from a bare `finally:` in+ # flush_worker, so raising here also skips the handoff cleanup, and an+ # undecodable file would otherwise be re-read on every flush forever.+ # Reported as delivered because there is nothing left to deliver and the+ # rest of the run should continue.+ try:+ claim.replace(claim.with_suffix(".corrupt"))+ except OSError:+ try:+ claim.unlink()+ except OSError:+ pass+ return 0, Trueexcept OSError:- return 0+ # Could not read it, which is not the same as having nothing to send.+ # The file is left exactly where it is: a vanished or briefly unreadable+ # claim is retryable, and quarantining it here would discard events over+ # a transient filesystem error. Reported as undelivered so the run stops+ # instead of counting a batch nothing was posted from as delivered.+ return 0, Falseevents = []for line in lines:try:@@ -497,11 +715,18 @@ def flush() -> int:if isinstance(value, dict) and value.get("event"):events.append(value)if not events:+ # Only delete when the file really is empty. A non-empty file that+ # parses to nothing is a torn write, and its contents are the unsent+ # remainder — deleting it is the data loss this PR exists to prevent.try:- claim.unlink()+ empty = claim.stat().st_size == 0+ except OSError:+ empty = True+ try:+ claim.replace(claim.with_suffix(".corrupt")) if not empty else claim.unlink()except OSError:pass- return 0+ return 0, Truedistinct_id, aliased_anonymous_id = resolve_distinct_id()if aliased_anonymous_id:@@ -520,10 +745,13 @@ def flush() -> int:sent = 0for start in range(0, len(events), BATCH_SIZE):+ chunk = events[start : start + BATCH_SIZE]batch = [{"event": event["event"],"distinct_id": distinct_id,+ # Carried through from record() so a resend can be collapsed.+ "uuid": event.get("uuid"),"timestamp": event.get("timestamp"),"properties": {# Fallback only: events recorded by a build before source@@ -535,16 +763,24 @@ def flush() -> int:**(event.get("properties") or {}),},}- for event in events[start : start + BATCH_SIZE]+ for event in chunk]if not _post({"api_key": POSTHOG_API_KEY, "batch": batch}, POSTHOG_BATCH_URL):- return sent- sent += len(batch)- try:- claim.unlink()- except OSError:- pass- return sent+ # Keep only what has not been delivered, and release the lease.+ # Previously the whole file was kept and the retry re-posted every+ # batch, including the ones that had already arrived.+ _release_claim(claim, events[start:])+ return sent, False+ sent += len(chunk)+ # Record progress and refresh the lease after each successful batch, so+ # a crash repeats at most one batch instead of the entire file. If the+ # rewrite fails the claim still holds delivered events, so stop rather+ # than carry on as though progress were recorded — continuing is how the+ # duplicate delivery this PR fixes would come back.+ if not _rewrite_claim(claim, events[start + len(chunk) :]):+ _release_claim(claim, events[start + len(chunk) :])+ return sent, False+ return sent, Truedef main() -> int: