Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

patch: Log stopped Code sessions and cleanup failures (#1948)

Ali Khokhard45d298d1933
diff --git a/src/free_claude_code/api/response_streams.py b/src/free_claude_code/api/response_streams.py
index 2a1777af..9c3ea29d 100644
--- a/src/free_claude_code/api/response_streams.py
+++ b/src/free_claude_code/api/response_streams.py
@@ -10,6 +10,7 @@ from collections.abc import (
from typing import Literal
from fastapi.responses import JSONResponse, Response, StreamingResponse
+from loguru import logger
from starlette.background import BackgroundTask
from starlette.responses import ContentStream
from starlette.types import Receive, Scope, Send
@@ -114,7 +115,7 @@ class ManagedStreamingResponse(StreamingResponse):
preserved_error=preserved_error,
)
except Exception as exc:
- _trace_response_cleanup_failure("close_body", exc)
+ _log_response_cleanup_failure("close_body", exc)
release = self._release
if release is None:
@@ -122,7 +123,7 @@ class ManagedStreamingResponse(StreamingResponse):
try:
await release()
except Exception as exc:
- _trace_response_cleanup_failure("release_resource", exc)
+ _log_response_cleanup_failure("release_resource", exc)
async def _wait_for_cleanup(task: asyncio.Task[None]) -> None:
@@ -134,7 +135,7 @@ async def _wait_for_cleanup(task: asyncio.Task[None]) -> None:
except asyncio.CancelledError as exc:
cancellation = exc
- # Ordinary defensive failures are trace-only; cancellation remains control flow.
+ # Ordinary cleanup failures preserve the response; cancellation remains control flow.
try:
task.result()
except asyncio.CancelledError:
@@ -142,20 +143,20 @@ async def _wait_for_cleanup(task: asyncio.Task[None]) -> None:
raise cancellation from None
raise
except Exception as exc:
- _trace_response_cleanup_failure("cleanup_task", exc)
+ _log_response_cleanup_failure("cleanup_task", exc)
if cancellation is not None:
raise cancellation
-def _trace_response_cleanup_failure(operation: str, exc: BaseException) -> None:
- trace_event(
+def _log_response_cleanup_failure(operation: str, exc: BaseException) -> None:
+ logger.bind(
stage="egress",
event="free_claude_code.api.response.cleanup_failed",
source="api",
operation=operation,
exc_type=type(exc).__name__,
- )
+ ).opt(exception=exc).warning("Response cleanup failed")
async def bind_response_lifetime(
diff --git a/src/free_claude_code/application/code_sessions/service.py b/src/free_claude_code/application/code_sessions/service.py
index b8ec855d..3651e3e4 100644
--- a/src/free_claude_code/application/code_sessions/service.py
+++ b/src/free_claude_code/application/code_sessions/service.py
@@ -183,9 +183,10 @@ class CodeService:
await self._stop(
owner.state.session.id, owner.state.run.id, shutting_down=True
)
- except Exception:
+ except Exception as exc:
+ self._mark_storage_failed(owner, "stop_run", exc)
await self._close_connection(owner)
- self._storage_failure(owner)
+ self._notify_storage_failure(owner)
jobs = [owner.job for owner in owners if owner.job is not None]
if jobs:
await asyncio.wait(jobs, timeout=5)
@@ -199,8 +200,9 @@ class CodeService:
"interrupted",
"FCC stopped before this turn finished.",
)
- except Exception:
- self._storage_failure(owner)
+ except Exception as exc:
+ self._mark_storage_failed(owner, "finish_run", exc)
+ self._notify_storage_failure(owner)
if self._jobs:
await asyncio.gather(*tuple(self._jobs), return_exceptions=True)
self._events.close()
@@ -978,8 +980,8 @@ class CodeService:
)
except CodeConflictError, CodeNotFoundError:
raise
- except Exception:
- owner.storage_failed = True
+ except Exception as exc:
+ self._mark_storage_failed(owner, "save_progress", exc)
if owner.failure_task is None:
owner.failure_task = self._job(self._halt_storage(owner))
raise
@@ -987,7 +989,7 @@ class CodeService:
async def _halt_storage(self, owner: _SessionRuntime) -> None:
await self._close_connection(owner)
- self._storage_failure(owner)
+ self._notify_storage_failure(owner)
def _detach_connection_locked(
self, owner: _SessionRuntime
@@ -1017,8 +1019,9 @@ class CodeService:
owner, owner.state.attach_thread(connection.thread_id)
)
await self._expire_prompts_locked(owner)
- except Exception:
- self._storage_failure(owner)
+ except Exception as exc:
+ self._mark_storage_failed(owner, "close_connection", exc)
+ self._notify_storage_failure(owner)
async def _fail(
self,
@@ -1049,11 +1052,24 @@ class CodeService:
else "failed"
)
await self._finish_locked(owner, status, message)
- except Exception:
- self._storage_failure(owner)
+ except Exception as exc:
+ self._mark_storage_failed(owner, "fail_run", exc)
+ self._notify_storage_failure(owner)
- def _storage_failure(self, owner: _SessionRuntime) -> None:
+ def _mark_storage_failed(
+ self, owner: _SessionRuntime, operation: str, error: Exception
+ ) -> None:
+ if owner.storage_failed:
+ return
owner.storage_failed = True
+ logger.bind(
+ event="code.storage_failed",
+ operation=operation,
+ session_id=owner.state.session.id,
+ run_id=owner.state.run.id if owner.state.run is not None else None,
+ ).opt(exception=error).error("Code session stopped after a storage failure")
+
+ def _notify_storage_failure(self, owner: _SessionRuntime) -> None:
owner.finished.set()
self._publish(
owner,
@@ -1104,8 +1120,11 @@ class CodeService:
progress = owner.state.deletion_failed(status, _error_message(exc))
try:
await self._commit_progress(owner, progress)
- except Exception:
- self._storage_failure(owner)
+ except Exception as save_error:
+ self._mark_storage_failed(
+ owner, "save_deletion_failure", save_error
+ )
+ self._notify_storage_failure(owner)
return
self._publish(owner, "session.updated")
if status == "delete_uncertain" and not reconcile and not native_complete:
diff --git a/src/free_claude_code/core/trace.py b/src/free_claude_code/core/trace.py
index 4d3bafef..26e322e5 100644
--- a/src/free_claude_code/core/trace.py
+++ b/src/free_claude_code/core/trace.py
@@ -1,4 +1,4 @@
-"""Structured DEBUG traces for end-to-end request / CLI / provider logging.
+"""Structured DEBUG traces and stream cleanup warnings.
Trace payloads are preserved in ``record.extra.trace_payload`` by Loguru.
Conversation and Claude Code prompts are logged verbatim unless values live under
@@ -73,7 +73,7 @@ async def close_stream_input(
close_error = await try_close_async_iterator(iterator)
if close_error is None:
return
- trace_event(
+ logger.bind(
stage="lifecycle",
event="stream.input.close_failed",
source=source,
@@ -82,7 +82,7 @@ async def close_stream_input(
preserved_exc_type=(
type(preserved_error).__name__ if preserved_error is not None else None
),
- )
+ ).opt(exception=close_error).warning("Stream input cleanup failed")
def extract_claude_session_id_from_headers(headers: Mapping[str, str]) -> str | None:
diff --git a/tests/api/test_response_streams.py b/tests/api/test_response_streams.py
index ce71e688..a53f93af 100644
--- a/tests/api/test_response_streams.py
+++ b/tests/api/test_response_streams.py
@@ -4,10 +4,11 @@ import asyncio
import json
from collections.abc import AsyncGenerator
from typing import Any, cast
-from unittest.mock import AsyncMock, patch
+from unittest.mock import AsyncMock
import pytest
from fastapi.responses import JSONResponse, StreamingResponse
+from loguru import logger
from starlette.types import Message, Scope
from free_claude_code.api.request_ids import RequestCorrelationMiddleware
@@ -573,7 +574,10 @@ async def test_cancellation_during_pre_start_error_cleanup_waits_for_close() ->
@pytest.mark.asyncio
-async def test_cleanup_failures_are_trace_only_and_do_not_replace_success() -> None:
+async def test_cleanup_failures_warn_once_and_do_not_replace_success(caplog) -> None:
+ close_error = RuntimeError("close detail")
+ release_error = RuntimeError("release detail")
+
class CloseFails:
def __init__(self) -> None:
self._yielded = False
@@ -588,33 +592,31 @@ async def test_cleanup_failures_are_trace_only_and_do_not_replace_success() -> N
return "ok"
async def aclose(self) -> None:
- raise RuntimeError("secret close detail")
+ raise close_error
- release = AsyncMock(side_effect=RuntimeError("secret release detail"))
+ release = AsyncMock(side_effect=release_error)
response = ManagedStreamingResponse(CloseFails())
await bind_response_lifetime(response, release)
- with (
- patch("free_claude_code.core.trace.trace_event") as close_trace,
- patch("free_claude_code.api.response_streams.trace_event") as release_trace,
- ):
+ with logger.contextualize(request_id="req_cleanup"):
messages = await _serve(response)
+ await response.aclose()
assert b"".join(message.get("body", b"") for message in messages) == b"ok"
- close_trace.assert_called_once()
- assert close_trace.call_args.kwargs["owner"] == "ManagedStreamingResponse"
- assert close_trace.call_args.kwargs["close_exc_type"] == "RuntimeError"
- assert release_trace.call_args.kwargs["operation"] == "release_resource"
- trace_blob = " ".join(
- str(call)
- for call in [*close_trace.call_args_list, *release_trace.call_args_list]
- )
- assert "secret close detail" not in trace_blob
- assert "secret release detail" not in trace_blob
+ warnings = [record for record in caplog.records if record.levelname == "WARNING"]
+ assert len(warnings) == 2
+ assert warnings[0].extra["event"] == "stream.input.close_failed"
+ assert warnings[0].extra["owner"] == "ManagedStreamingResponse"
+ assert warnings[0].exc_info[1] is close_error
+ assert warnings[1].extra["operation"] == "release_resource"
+ assert warnings[1].exc_info[1] is release_error
+ assert all(record.extra["request_id"] == "req_cleanup" for record in warnings)
+ assert "aclose" in caplog.text
+ release.assert_awaited_once()
@pytest.mark.asyncio
-async def test_body_close_cancellation_propagates_without_releasing() -> None:
+async def test_body_close_cancellation_propagates_without_releasing(caplog) -> None:
class CloseIsCancelled:
def __aiter__(self):
return self
@@ -633,6 +635,7 @@ async def test_body_close_cancellation_propagates_without_releasing() -> None:
await response.aclose()
release.assert_not_awaited()
+ assert not [record for record in caplog.records if record.levelname == "WARNING"]
@pytest.mark.asyncio
diff --git a/tests/application/test_code_sessions.py b/tests/application/test_code_sessions.py
index caae66aa..62c037e3 100644
--- a/tests/application/test_code_sessions.py
+++ b/tests/application/test_code_sessions.py
@@ -1286,19 +1286,21 @@ async def test_shutdown_settles_run_when_native_interrupt_never_acknowledges(cod
@pytest.mark.asyncio
async def test_storage_failure_retires_native_work_even_with_pending_prompt(
- code, monkeypatch
+ code, monkeypatch, caplog
):
service, harness, _ = code
session = await session_for(code)
- await service.send(
+ run = await service.send(
session.id, new_id(), session.revision, "prompt", expected_epoch=service.epoch
)
await harness.started.wait()
connection = harness.connections[0]
await connection.prompt(1)
+ error = CodeUnavailableError("Code disk unavailable")
+
async def fail_save(*args, **kwargs):
- raise CodeUnavailableError("Code disk unavailable")
+ raise error
monkeypatch.setattr(service._store, "save_progress", fail_save)
await connection.text("turn-1", "text", "unsaved", complete=True)
@@ -1306,6 +1308,74 @@ async def test_storage_failure_retires_native_work_even_with_pending_prompt(
assert connection.closed
with pytest.raises(CodeUnavailableError):
await service.get_detail(session.id)
+ await service.close()
+ failures = [record for record in caplog.records if record.levelname == "ERROR"]
+ assert len(failures) == 1
+ assert failures[0].extra == {
+ "event": "code.storage_failed",
+ "operation": "save_progress",
+ "session_id": session.id,
+ "run_id": run.id,
+ }
+ assert failures[0].exc_info[1] is error
+ assert "fail_save" in caplog.text
+
+
+async def test_shutdown_storage_read_failure_logs_original_cause(
+ code, monkeypatch, caplog
+):
+ service, harness, _ = code
+ session = await session_for(code)
+ run = await service.send(
+ session.id, new_id(), session.revision, "work", expected_epoch=service.epoch
+ )
+ await harness.started.wait()
+ error = CodeUnavailableError("Cannot read run during shutdown")
+ monkeypatch.setattr(service._store, "get_run", AsyncMock(side_effect=error))
+
+ await service.close()
+
+ assert harness.connections[0].closed
+ failures = [record for record in caplog.records if record.levelname == "ERROR"]
+ assert len(failures) == 1
+ assert failures[0].extra["operation"] == "stop_run"
+ assert failures[0].extra["session_id"] == session.id
+ assert failures[0].extra["run_id"] == run.id
+ assert failures[0].exc_info[1] is error
+
+
[email protected]("conflict", [False, True])
+async def test_delete_before_first_run_logs_only_terminal_storage_failure(
+ code, monkeypatch, caplog, conflict
+):
+ service, _, _ = code
+ session = await session_for(code)
+ error = (
+ CodeConflictError("Revision changed")
+ if conflict
+ else CodeUnavailableError("Cannot save deletion")
+ )
+ save = service._store.save_progress
+ monkeypatch.setattr(service._store, "save_progress", AsyncMock(side_effect=error))
+
+ with pytest.raises(type(error)):
+ await service.delete_session(session.id, session.revision)
+ if conflict:
+ monkeypatch.setattr(service._store, "save_progress", save)
+ assert (await service.get_detail(session.id)).session == session
+ else:
+ await asyncio.wait_for(service.wait_idle(session.id), 3)
+ await service.close()
+
+ failures = [record for record in caplog.records if record.levelname == "ERROR"]
+ assert len(failures) == (0 if conflict else 1)
+ if not conflict:
+ assert failures[0].extra["session_id"] == session.id
+ assert failures[0].extra["run_id"] is None
+ assert failures[0].extra["operation"] == "save_progress"
+ assert failures[0].exc_info[1] is error
@pytest.mark.asyncio
diff --git a/tests/application/test_execution.py b/tests/application/test_execution.py
index 8426b331..0063426e 100644
--- a/tests/application/test_execution.py
+++ b/tests/application/test_execution.py
@@ -1120,7 +1120,7 @@ async def test_provider_cleanup_cannot_delay_fallback_past_progress_deadline() -
@pytest.mark.asyncio
-async def test_cleanup_failure_trace_names_preserved_provider_failure() -> None:
+async def test_cleanup_warning_names_preserved_provider_failure(caplog) -> None:
failure = _execution_failure("primary overloaded")
provider = CloseControlledProvider(
failure,
@@ -1132,20 +1132,16 @@ async def test_cleanup_failure_trace_names_preserved_provider_failure() -> None:
request_id="req_close_failure_trace",
)
- with (
- patch("free_claude_code.core.trace.trace_event") as trace_mock,
- pytest.raises(ExecutionFailure) as exc_info,
- ):
+ with pytest.raises(ExecutionFailure) as exc_info:
await anext(stream)
assert exc_info.value is failure
- close_trace = next(
- call.kwargs
- for call in trace_mock.call_args_list
- if call.kwargs.get("event") == "stream.input.close_failed"
- )
- assert close_trace["close_exc_type"] == "RuntimeError"
- assert close_trace["preserved_exc_type"] == "ExecutionFailure"
+ warnings = [record for record in caplog.records if record.levelname == "WARNING"]
+ assert len(warnings) == 1
+ assert warnings[0].extra["event"] == "stream.input.close_failed"
+ assert warnings[0].extra["close_exc_type"] == "RuntimeError"
+ assert warnings[0].extra["preserved_exc_type"] == "ExecutionFailure"
+ assert str(warnings[0].exc_info[1]) == "close failed"
@pytest.mark.asyncio
diff --git a/tests/core/test_trace.py b/tests/core/test_trace.py
index deccfb80..47bbd7f8 100644
--- a/tests/core/test_trace.py
+++ b/tests/core/test_trace.py
@@ -160,12 +160,16 @@ async def test_traced_async_stream_logs_real_exception(tmp_path) -> None:
assert interrupted[0]["outcome"] == "error"
assert interrupted[0]["exc_type"] == "RuntimeError"
close_failed = [
- row for row in rows if row.get("event") == "stream.input.close_failed"
+ row["record"]
+ for row in _json_log_rows(log_file)
+ if row["record"]["extra"].get("event") == "stream.input.close_failed"
]
assert len(close_failed) == 1
- assert close_failed[0]["owner"] == "traced_async_stream"
- assert close_failed[0]["close_exc_type"] == "RuntimeError"
- assert close_failed[0]["preserved_exc_type"] == "RuntimeError"
+ assert close_failed[0]["level"]["name"] == "WARNING"
+ assert close_failed[0]["exception"]["value"] == "close boom"
+ assert close_failed[0]["extra"]["owner"] == "traced_async_stream"
+ assert close_failed[0]["extra"]["close_exc_type"] == "RuntimeError"
+ assert close_failed[0]["extra"]["preserved_exc_type"] == "RuntimeError"
@pytest.mark.asyncio