AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
hotfix(backend/copilot): refactor transcript to SDK-based atomic full-context model (#12318)
Zamil Majdyd56452898ad3
diff --git a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.pyindex edac1f8907..e2dc2062d2 100644--- a/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py+++ b/autogpt_platform/backend/backend/copilot/sdk/security_hooks.py@@ -127,7 +127,6 @@ def create_security_hooks(sdk_cwd: str | None = None,max_subtasks: int = 3,on_compact: Callable[[], None] | None = None,- on_stop: Callable[[str, str], None] | None = None,) -> dict[str, Any]:"""Create the security hooks configuration for Claude Agent SDK.@@ -136,15 +135,12 @@ def create_security_hooks(- PostToolUse: Log successful tool executions- PostToolUseFailure: Log and handle failed tool executions- PreCompact: Log context compaction events (SDK handles compaction automatically)- - Stop: Capture transcript path for stateless resume (when *on_stop* is provided)Args:user_id: Current user ID for isolation validationsdk_cwd: SDK working directory for workspace-scoped tool validationmax_subtasks: Maximum concurrent Task (sub-agent) spawns allowed per session- on_stop: Callback ``(transcript_path, sdk_session_id)`` invoked when- the SDK finishes processing — used to read the JSONL transcript- before the CLI process exits.+ on_compact: Callback invoked when SDK starts compacting context.Returns:Hooks configuration dict for ClaudeAgentOptions@@ -311,30 +307,6 @@ def create_security_hooks(on_compact()return cast(SyncHookJSONOutput, {})- # --- Stop hook: capture transcript path for stateless resume ---- async def stop_hook(- input_data: HookInput,- tool_use_id: str | None,- context: HookContext,- ) -> SyncHookJSONOutput:- """Capture transcript path when SDK finishes processing.-- The Stop hook fires while the CLI process is still alive, giving us- a reliable window to read the JSONL transcript before SIGTERM.- """- _ = context, tool_use_id- transcript_path = cast(str, input_data.get("transcript_path", ""))- sdk_session_id = cast(str, input_data.get("session_id", ""))-- if transcript_path and on_stop:- logger.info(- f"[SDK] Stop hook: transcript_path={transcript_path}, "- f"sdk_session_id={sdk_session_id[:12]}..."- )- on_stop(transcript_path, sdk_session_id)-- return cast(SyncHookJSONOutput, {})-hooks: dict[str, Any] = {"PreToolUse": [HookMatcher(matcher="*", hooks=[pre_tool_use_hook])],"PostToolUse": [HookMatcher(matcher="*", hooks=[post_tool_use_hook])],@@ -344,9 +316,6 @@ def create_security_hooks("PreCompact": [HookMatcher(matcher="*", hooks=[pre_compact_hook])],}- if on_stop is not None:- hooks["Stop"] = [HookMatcher(matcher=None, hooks=[stop_hook])]-return hooksexcept ImportError:# Fallback for when SDK isn't available - return empty hooksdiff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 729e14eb84..e3c0f0bb94 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -12,7 +12,6 @@ import subprocessimport sysimport uuidfrom collections.abc import AsyncGenerator-from dataclasses import dataclassfrom typing import Any, castimport openai@@ -21,6 +20,9 @@ from claude_agent_sdk import (ClaudeAgentOptions,ClaudeSDKClient,ResultMessage,+ TextBlock,+ ThinkingBlock,+ ToolResultBlock,ToolUseBlock,)from langfuse import propagate_attributes@@ -74,11 +76,11 @@ from .tool_adapter import (from .transcript import (cleanup_cli_project_dir,download_transcript,- read_transcript_file,upload_transcript,validate_transcript,write_transcript_to_tempfile,)+from .transcript_builder import TranscriptBuilderlogger = logging.getLogger(__name__)config = ChatConfig()@@ -137,19 +139,6 @@ _setup_langfuse_otel()_background_tasks: set[asyncio.Task[Any]] = set()-@dataclass-class CapturedTranscript:- """Info captured by the SDK Stop hook for stateless --resume."""-- path: str = ""- sdk_session_id: str = ""- raw_content: str = ""-- @property- def available(self) -> bool:- return bool(self.path)--_SDK_CWD_PREFIX = WORKSPACE_PREFIX# Heartbeat interval — keep SSE alive through proxies/LBs during tool execution.@@ -451,6 +440,49 @@ def _cleanup_sdk_tool_results(cwd: str) -> None:pass+def _format_sdk_content_blocks(blocks: list) -> list[dict[str, Any]]:+ """Convert SDK content blocks to transcript format.++ Handles TextBlock, ToolUseBlock, ToolResultBlock, and ThinkingBlock.+ Unknown block types are logged and skipped.+ """+ result: list[dict[str, Any]] = []+ for block in blocks or []:+ if isinstance(block, TextBlock):+ result.append({"type": "text", "text": block.text})+ elif isinstance(block, ToolUseBlock):+ result.append(+ {+ "type": "tool_use",+ "id": block.id,+ "name": block.name,+ "input": block.input,+ }+ )+ elif isinstance(block, ToolResultBlock):+ result.append(+ {+ "type": "tool_result",+ "tool_use_id": block.tool_use_id,+ "content": block.content,+ }+ )+ elif isinstance(block, ThinkingBlock):+ result.append(+ {+ "type": "thinking",+ "thinking": block.thinking,+ "signature": block.signature,+ }+ )+ else:+ logger.warning(+ f"[SDK] Unknown content block type: {type(block).__name__}. "+ f"This may indicate a new SDK version with additional block types."+ )+ return result++async def _compress_messages(messages: list[ChatMessage],) -> tuple[list[ChatMessage], bool]:@@ -806,6 +838,11 @@ async def stream_chat_completion_sdk(user_id=user_id, session_id=session_id, message_length=len(message))+ # Structured log prefix: [SDK][<session>][T<turn>]+ # Turn = number of user messages (1-based), computed AFTER appending the new message.+ turn = sum(1 for m in session.messages if m.role == "user")+ log_prefix = f"[SDK][{session_id[:12]}][T{turn}]"+session = await upsert_chat_session(session)# Generate title for new sessions (first user message)@@ -823,10 +860,11 @@ async def stream_chat_completion_sdk(message_id = str(uuid.uuid4())stream_id = str(uuid.uuid4())stream_completed = False+ ended_with_stream_error = Falsee2b_sandbox = Noneuse_resume = Falseresume_file: str | None = None- captured_transcript = CapturedTranscript()+ transcript_builder = TranscriptBuilder()sdk_cwd = ""# Acquire stream lock to prevent concurrent streams to the same session@@ -841,7 +879,7 @@ async def stream_chat_completion_sdk(if lock_owner != stream_id:# Another stream is activelogger.warning(- f"[SDK] Session {session_id} already has an active stream: {lock_owner}"+ f"{log_prefix} Session already has an active stream: {lock_owner}")yield StreamError(errorText="Another stream is already active for this session. "@@ -865,7 +903,7 @@ async def stream_chat_completion_sdk(sdk_cwd = _make_sdk_cwd(session_id)os.makedirs(sdk_cwd, exist_ok=True)except (ValueError, OSError) as e:- logger.error("[SDK] [%s] Invalid SDK cwd: %s", session_id[:12], e)+ logger.error("%s Invalid SDK cwd: %s", log_prefix, e)yield StreamError(errorText="Unable to initialize working directory.",code="sdk_cwd_error",@@ -909,12 +947,13 @@ async def stream_chat_completion_sdk():return Nonetry:- return await download_transcript(user_id, session_id)+ return await download_transcript(+ user_id, session_id, log_prefix=log_prefix+ )except Exception as transcript_err:logger.warning(- "[SDK] [%s] Transcript download failed, continuing without "- "--resume: %s",- session_id[:12],+ "%s Transcript download failed, continuing without " "--resume: %s",+ log_prefix,transcript_err,)return None@@ -936,11 +975,18 @@ async def stream_chat_completion_sdk(transcript_msg_count = 0if dl:is_valid = validate_transcript(dl.content)+ dl_lines = dl.content.strip().split("\n") if dl.content else []+ logger.info(+ "%s Downloaded transcript: %dB, %d lines, " "msg_count=%d, valid=%s",+ log_prefix,+ len(dl.content),+ len(dl_lines),+ dl.message_count,+ is_valid,+ )if is_valid:- logger.info(- f"[SDK] Transcript available for session {session_id}: "- f"{len(dl.content)}B, msg_count={dl.message_count}"- )+ # Load previous FULL context into builder+ transcript_builder.load_previous(dl.content, log_prefix=log_prefix)resume_file = write_transcript_to_tempfile(dl.content, session_id, sdk_cwd)@@ -948,16 +994,14 @@ async def stream_chat_completion_sdk(use_resume = Truetranscript_msg_count = dl.message_countlogger.debug(- f"[SDK] Using --resume ({len(dl.content)}B, "+ f"{log_prefix} Using --resume ({len(dl.content)}B, "f"msg_count={transcript_msg_count})")else:- logger.warning(- f"[SDK] Transcript downloaded but invalid for {session_id}"- )+ logger.warning(f"{log_prefix} Transcript downloaded but invalid")elif config.claude_agent_use_resume and user_id and len(session.messages) > 1:logger.warning(- f"[SDK] No transcript available for {session_id} "+ f"{log_prefix} No transcript available "f"({len(session.messages)} messages in session)")@@ -979,25 +1023,6 @@ async def stream_chat_completion_sdk(sdk_model = _resolve_sdk_model()- # --- Transcript capture via Stop hook ---- # Read the file content immediately — the SDK may clean up- # the file before our finally block runs.- def _on_stop(transcript_path: str, sdk_session_id: str) -> None:- captured_transcript.path = transcript_path- captured_transcript.sdk_session_id = sdk_session_id- content = read_transcript_file(transcript_path)- if content:- captured_transcript.raw_content = content- logger.info(- f"[SDK] Stop hook: captured {len(content)}B from "- f"{transcript_path}"- )- else:- logger.warning(- f"[SDK] Stop hook: transcript file empty/missing at "- f"{transcript_path}"- )-# Track SDK-internal compaction (PreCompact hook → start, next msg → end)compaction = CompactionTracker()@@ -1005,7 +1030,6 @@ async def stream_chat_completion_sdk(user_id,sdk_cwd=sdk_cwd,max_subtasks=config.claude_agent_max_subtasks,- on_stop=_on_stop if config.claude_agent_use_resume else None,on_compact=compaction.on_compact,)@@ -1040,7 +1064,10 @@ async def stream_chat_completion_sdk(session_id=session_id,trace_name="copilot-sdk",tags=["sdk"],- metadata={"resume": str(use_resume)},+ metadata={+ "resume": str(use_resume),+ "conversation_turn": str(turn),+ },)_otel_ctx.__enter__()@@ -1074,9 +1101,9 @@ async def stream_chat_completion_sdk(query_message = f"{query_message}\n\n{attachments.hint}"logger.info(- "[SDK] [%s] Sending query — resume=%s, total_msgs=%d, "+ "%s Sending query — resume=%s, total_msgs=%d, ""query_len=%d, attached_files=%d, image_blocks=%d",- session_id[:12],+ log_prefix,use_resume,len(session.messages),len(query_message),@@ -1105,8 +1132,13 @@ async def stream_chat_completion_sdk(await client._transport.write( # noqa: SLF001json.dumps(user_msg) + "\n")+ # Capture user message in transcript (multimodal)+ transcript_builder.add_user_message(content=content_blocks)else:await client.query(query_message, session_id=session_id)+ # Capture actual user message in transcript (not the engineered query)+ # query_message may include context wrappers, but transcript needs raw input+ transcript_builder.add_user_message(content=current_message)assistant_response = ChatMessage(role="assistant", content="")accumulated_tool_calls: list[dict[str, Any]] = []@@ -1150,8 +1182,8 @@ async def stream_chat_completion_sdk(sdk_msg = done.pop().result()except StopAsyncIteration:logger.info(- "[SDK] [%s] Stream ended normally (StopAsyncIteration)",- session_id[:12],+ "%s Stream ended normally (StopAsyncIteration)",+ log_prefix,)breakexcept Exception as stream_err:@@ -1160,8 +1192,8 @@ async def stream_chat_completion_sdk(# so the session can still be saved and the# frontend gets a clean finish.logger.error(- "[SDK] [%s] Stream error from SDK: %s",- session_id[:12],+ "%s Stream error from SDK: %s",+ log_prefix,stream_err,exc_info=True,)@@ -1173,9 +1205,9 @@ async def stream_chat_completion_sdk(breaklogger.info(- "[SDK] [%s] Received: %s %s "+ "%s Received: %s %s ""(unresolved=%d, current=%d, resolved=%d)",- session_id[:12],+ log_prefix,type(sdk_msg).__name__,getattr(sdk_msg, "subtype", ""),len(adapter.current_tool_calls)@@ -1184,6 +1216,15 @@ async def stream_chat_completion_sdk(len(adapter.resolved_tool_calls),)+ # Capture SDK messages in transcript+ if isinstance(sdk_msg, AssistantMessage):+ content_blocks = _format_sdk_content_blocks(sdk_msg.content)+ model_name = getattr(sdk_msg, "model", "")+ transcript_builder.add_assistant_message(+ content_blocks=content_blocks,+ model=model_name,+ )+# Race-condition fix: SDK hooks (PostToolUse) are# executed asynchronously via start_soon() — the next# message can arrive before the hook stashes output.@@ -1210,10 +1251,10 @@ async def stream_chat_completion_sdk(await asyncio.sleep(0)else:logger.warning(- "[SDK] [%s] Timed out waiting for "+ "%s Timed out waiting for ""PostToolUse hook stash ""(%d unresolved tool calls)",- session_id[:12],+ log_prefix,len(adapter.current_tool_calls)- len(adapter.resolved_tool_calls),)@@ -1221,9 +1262,9 @@ async def stream_chat_completion_sdk(# Log ResultMessage details for debuggingif isinstance(sdk_msg, ResultMessage):logger.info(- "[SDK] [%s] Received: ResultMessage %s "+ "%s Received: ResultMessage %s ""(unresolved=%d, current=%d, resolved=%d)",- session_id[:12],+ log_prefix,sdk_msg.subtype,len(adapter.current_tool_calls)- len(adapter.resolved_tool_calls),@@ -1232,8 +1273,8 @@ async def stream_chat_completion_sdk()if sdk_msg.subtype in ("error", "error_during_execution"):logger.error(- "[SDK] [%s] SDK execution failed with error: %s",- session_id[:12],+ "%s SDK execution failed with error: %s",+ log_prefix,sdk_msg.result or "(no error message provided)",)@@ -1258,8 +1299,8 @@ async def stream_chat_completion_sdk(out_len = len(str(response.output))extra = f", output_len={out_len}"logger.info(- "[SDK] [%s] Tool event: %s, tool=%s%s",- session_id[:12],+ "%s Tool event: %s, tool=%s%s",+ log_prefix,type(response).__name__,getattr(response, "toolName", "N/A"),extra,@@ -1268,8 +1309,8 @@ async def stream_chat_completion_sdk(# Log errors being sent to frontendif isinstance(response, StreamError):logger.error(- "[SDK] [%s] Sending error to frontend: %s (code=%s)",- session_id[:12],+ "%s Sending error to frontend: %s (code=%s)",+ log_prefix,response.errorText,response.code,)@@ -1335,8 +1376,8 @@ async def stream_chat_completion_sdk(# server shutdown). Log and let the safety-net / finally# blocks handle cleanup.logger.warning(- "[SDK] [%s] Streaming loop cancelled (asyncio.CancelledError)",- session_id[:12],+ "%s Streaming loop cancelled (asyncio.CancelledError)",+ log_prefix,)raisefinally:@@ -1350,7 +1391,8 @@ async def stream_chat_completion_sdk(except (asyncio.CancelledError, StopAsyncIteration):# Expected: task was cancelled or exhausted during cleanuplogger.info(- "[SDK] Pending __anext__ task completed during cleanup"+ "%s Pending __anext__ task completed during cleanup",+ log_prefix,)# Safety net: if tools are still unresolved after the@@ -1359,9 +1401,9 @@ async def stream_chat_completion_sdk(# them now so the frontend stops showing spinners.if adapter.has_unresolved_tool_calls:logger.warning(- "[SDK] [%s] %d unresolved tool(s) after stream loop — "+ "%s %d unresolved tool(s) after stream loop — ""flushing as safety net",- session_id[:12],+ log_prefix,len(adapter.current_tool_calls) - len(adapter.resolved_tool_calls),)safety_responses: list[StreamBaseResponse] = []@@ -1372,8 +1414,8 @@ async def stream_chat_completion_sdk((StreamToolInputAvailable, StreamToolOutputAvailable),):logger.info(- "[SDK] [%s] Safety flush: %s, tool=%s",- session_id[:12],+ "%s Safety flush: %s, tool=%s",+ log_prefix,type(response).__name__,getattr(response, "toolName", "N/A"),)@@ -1386,8 +1428,8 @@ async def stream_chat_completion_sdk(# StreamFinish is published by mark_session_completed in the processor.if not stream_completed and not ended_with_stream_error:logger.info(- "[SDK] [%s] Stream ended without ResultMessage (stopped by user)",- session_id[:12],+ "%s Stream ended without ResultMessage (stopped by user)",+ log_prefix,)closing_responses: list[StreamBaseResponse] = []adapter._end_text_if_open(closing_responses)@@ -1408,69 +1450,36 @@ async def stream_chat_completion_sdk() and not has_appended_assistant:session.messages.append(assistant_response)- # --- Upload transcript for next-turn --resume ---- # After async with the SDK task group has exited, so the Stop- # hook has already fired and the CLI has been SIGTERMed. The- # CLI uses appendFileSync, so all writes are safely on disk.- if config.claude_agent_use_resume and user_id:- # With --resume the CLI appends to the resume file (most- # complete). Otherwise use the Stop hook path.- if use_resume and resume_file:- raw_transcript = read_transcript_file(resume_file)- logger.debug("[SDK] Transcript source: resume file")- elif captured_transcript.path:- raw_transcript = read_transcript_file(captured_transcript.path)- logger.debug(- "[SDK] Transcript source: stop hook (%s), read result: %s",- captured_transcript.path,- f"{len(raw_transcript)}B" if raw_transcript else "None",- )- else:- raw_transcript = None+ # Transcript upload is handled exclusively in the finally block+ # to avoid double-uploads (the success path used to upload the+ # old resume file, then the finally block overwrote it with the+ # stop hook content — which could be smaller after compaction).- if not raw_transcript:- logger.debug(- "[SDK] No usable transcript — CLI file had no "- "conversation entries (expected for first turn "- "without --resume)"- )-- if raw_transcript:- # Shield the upload from generator cancellation so a- # client disconnect / page refresh doesn't lose the- # transcript. The upload must finish even if the SSE- # connection is torn down.- await asyncio.shield(- _try_upload_transcript(- user_id,- session_id,- raw_transcript,- message_count=len(session.messages),- )- )-- logger.info(- "[SDK] [%s] Stream completed successfully with %d messages",- session_id[:12],- len(session.messages),- )+ if ended_with_stream_error:+ logger.warning(+ "%s Stream ended with SDK error after %d messages",+ log_prefix,+ len(session.messages),+ )+ else:+ logger.info(+ "%s Stream completed successfully with %d messages",+ log_prefix,+ len(session.messages),+ )except BaseException as e:# Catch BaseException to handle both Exception and CancelledError# (CancelledError inherits from BaseException in Python 3.8+)if isinstance(e, asyncio.CancelledError):- logger.warning("[SDK] [%s] Session cancelled", session_id[:12])+ logger.warning("%s Session cancelled", log_prefix)error_msg = "Operation cancelled"else:error_msg = str(e) or type(e).__name__# SDK cleanup RuntimeError is expected during cancellation, log as warningif isinstance(e, RuntimeError) and "cancel scope" in str(e):- logger.warning(- "[SDK] [%s] SDK cleanup error: %s", session_id[:12], error_msg- )+ logger.warning("%s SDK cleanup error: %s", log_prefix, error_msg)else:- logger.error(- f"[SDK] [%s] Error: {error_msg}", session_id[:12], exc_info=True- )+ logger.error("%s Error: %s", log_prefix, error_msg, exc_info=True)# Append error marker to session (non-invasive text parsing approach)# The finally block will persist the session with this error marker@@ -1481,8 +1490,8 @@ async def stream_chat_completion_sdk())logger.debug(- "[SDK] [%s] Appended error marker, will be persisted in finally",- session_id[:12],+ "%s Appended error marker, will be persisted in finally",+ log_prefix,)# Yield StreamError for immediate feedback (only for non-cancellation errors)@@ -1514,47 +1523,61 @@ async def stream_chat_completion_sdk(try:await asyncio.shield(upsert_chat_session(session))logger.info(- "[SDK] [%s] Session persisted in finally with %d messages",- session_id[:12],+ "%s Session persisted in finally with %d messages",+ log_prefix,len(session.messages),)except Exception as persist_err:logger.error(- "[SDK] [%s] Failed to persist session in finally: %s",- session_id[:12],+ "%s Failed to persist session in finally: %s",+ log_prefix,persist_err,exc_info=True,)# --- Upload transcript for next-turn --resume ---# This MUST run in finally so the transcript is uploaded even when- # the streaming loop raises an exception. The CLI uses- # appendFileSync, so whatever was written before the error/SIGTERM- # is safely on disk and still useful for the next turn.- if config.claude_agent_use_resume and user_id:+ # the streaming loop raises an exception.+ # The transcript represents the COMPLETE active context (atomic).+ if config.claude_agent_use_resume and user_id and session is not None:try:- # Prefer content captured in the Stop hook (read before- # cleanup removes the file). Fall back to the resume- # file when the stop hook didn't fire (e.g. error before- # completion) so we don't lose the prior transcript.- raw_transcript = captured_transcript.raw_content or None- if not raw_transcript and use_resume and resume_file:- raw_transcript = read_transcript_file(resume_file)-- if raw_transcript and session is not None:+ # Build complete transcript from captured SDK messages+ transcript_content = transcript_builder.to_jsonl()++ if not transcript_content:+ logger.warning(+ "%s No transcript to upload (builder empty)", log_prefix+ )+ elif not validate_transcript(transcript_content):+ logger.warning(+ "%s Transcript invalid, skipping upload (entries=%d)",+ log_prefix,+ transcript_builder.entry_count,+ )+ else:+ logger.info(+ "%s Uploading complete transcript (entries=%d, bytes=%d)",+ log_prefix,+ transcript_builder.entry_count,+ len(transcript_content),+ )+ # Shield upload from cancellation - let it complete even if+ # the finally block is interrupted. No timeout to avoid race+ # conditions where backgrounded uploads overwrite newer transcripts.await asyncio.shield(- _try_upload_transcript(- user_id,- session_id,- raw_transcript,+ upload_transcript(+ user_id=user_id,+ session_id=session_id,+ content=transcript_content,message_count=len(session.messages),+ log_prefix=log_prefix,))- else:- logger.warning(f"[SDK] No transcript to upload for {session_id}")except Exception as upload_err:logger.error(- f"[SDK] Transcript upload failed in finally: {upload_err}",+ "%s Transcript upload failed in finally: %s",+ log_prefix,+ upload_err,exc_info=True,)@@ -1565,33 +1588,6 @@ async def stream_chat_completion_sdk(await lock.release()-async def _try_upload_transcript(- user_id: str,- session_id: str,- raw_content: str,- message_count: int = 0,-) -> bool:- """Strip progress entries and upload transcript (with timeout).-- Returns True if the upload completed without error.- """- try:- async with asyncio.timeout(30):- await upload_transcript(- user_id, session_id, raw_content, message_count=message_count- )- return True- except asyncio.TimeoutError:- logger.warning(f"[SDK] Transcript upload timed out for {session_id}")- return False- except Exception as e:- logger.error(- f"[SDK] Failed to upload transcript for {session_id}: {e}",- exc_info=True,- )- return False--async def _update_title_async(session_id: str, message: str, user_id: str | None = None) -> None:@@ -1600,7 +1596,7 @@ async def _update_title_async(title = await _generate_session_title(message, user_id=user_id, session_id=session_id)- if title:+ if title and user_id:await update_session_title(session_id, title)logger.debug(f"[SDK] Generated title for {session_id}: {title}")except Exception as e:diff --git a/autogpt_platform/backend/backend/copilot/sdk/transcript.py b/autogpt_platform/backend/backend/copilot/sdk/transcript.pyindex 257e20e869..31b6c710b7 100644--- a/autogpt_platform/backend/backend/copilot/sdk/transcript.py+++ b/autogpt_platform/backend/backend/copilot/sdk/transcript.py@@ -10,13 +10,14 @@ Storage is handled via ``WorkspaceStorageBackend`` (GCS in prod, localfilesystem for self-hosted) — no DB column needed."""-import jsonimport loggingimport osimport reimport timefrom dataclasses import dataclass+from backend.util import json+logger = logging.getLogger(__name__)# UUIDs are hex + hyphens; strip everything else to prevent path injection.@@ -58,41 +59,37 @@ def strip_progress_entries(content: str) -> str:Removes entries whose ``type`` is in ``STRIPPABLE_TYPES`` and reparentsany remaining child entries so the ``parentUuid`` chain stays intact.Typically reduces transcript size by ~30%.++ Entries that are not stripped or reparented are kept as their original+ raw JSON line to avoid unnecessary re-serialization that changes+ whitespace or key ordering."""lines = content.strip().split("\n")- entries: list[dict] = []+ # Parse entries, keeping the original line alongside the parsed dict.+ parsed: list[tuple[str, dict | None]] = []for line in lines:- try:- entries.append(json.loads(line))- except json.JSONDecodeError:- # Keep unparseable lines as-is (safety)- entries.append({"_raw": line})+ parsed.append((line, json.loads(line, fallback=None)))+ # First pass: identify stripped UUIDs and build parent map.stripped_uuids: set[str] = set()uuid_to_parent: dict[str, str] = {}- kept: list[dict] = []- for entry in entries:- if "_raw" in entry:- kept.append(entry)+ for _line, entry in parsed:+ if not isinstance(entry, dict):continueuid = entry.get("uuid", "")parent = entry.get("parentUuid", "")- entry_type = entry.get("type", "")-if uid:uuid_to_parent[uid] = parent-- if entry_type in STRIPPABLE_TYPES:- if uid:- stripped_uuids.add(uid)- else:- kept.append(entry)-- # Reparent: walk up chain through stripped entries to find surviving ancestor- for entry in kept:- if "_raw" in entry:+ if entry.get("type", "") in STRIPPABLE_TYPES and uid:+ stripped_uuids.add(uid)++ # Second pass: keep non-stripped entries, reparenting where needed.+ # Preserve original line when no reparenting is required.+ reparented: set[str] = set()+ for _line, entry in parsed:+ if not isinstance(entry, dict):continueparent = entry.get("parentUuid", "")original_parent = parent@@ -100,63 +97,32 @@ def strip_progress_entries(content: str) -> str:parent = uuid_to_parent.get(parent, "")if parent != original_parent:entry["parentUuid"] = parent+ uid = entry.get("uuid", "")+ if uid:+ reparented.add(uid)result_lines: list[str] = []- for entry in kept:- if "_raw" in entry:- result_lines.append(entry["_raw"])- else:+ for line, entry in parsed:+ if not isinstance(entry, dict):+ result_lines.append(line)+ continue+ if entry.get("type", "") in STRIPPABLE_TYPES:+ continue+ uid = entry.get("uuid", "")+ if uid in reparented:+ # Re-serialize only entries whose parentUuid was changed.result_lines.append(json.dumps(entry, separators=(",", ":")))+ else:+ result_lines.append(line)return "\n".join(result_lines) + "\n"# ----------------------------------------------------------------------------# Local file I/O (read from CLI's JSONL, write temp file for --resume)+# Local file I/O (write temp file for --resume)# ----------------------------------------------------------------------------def read_transcript_file(transcript_path: str) -> str | None:- """Read a JSONL transcript file from disk.-- Returns the raw JSONL content, or ``None`` if the file is missing, empty,- or only contains metadata (≤2 lines with no conversation messages).- """- if not transcript_path or not os.path.isfile(transcript_path):- logger.debug(f"[Transcript] File not found: {transcript_path}")- return None-- try:- with open(transcript_path) as f:- content = f.read()-- if not content.strip():- logger.debug("[Transcript] File is empty: %s", transcript_path)- return None-- lines = content.strip().split("\n")-- # Validate that the transcript has real conversation content- # (not just metadata like queue-operation entries).- if not validate_transcript(content):- logger.debug(- "[Transcript] No conversation content (%d lines) in %s",- len(lines),- transcript_path,- )- return None-- logger.info(- f"[Transcript] Read {len(lines)} lines, "- f"{len(content)} bytes from {transcript_path}"- )- return content-- except (json.JSONDecodeError, OSError) as e:- logger.warning(f"[Transcript] Failed to read {transcript_path}: {e}")- return None--def _sanitize_id(raw_id: str, max_len: int = 36) -> str:"""Sanitize an ID for safe use in file paths.@@ -171,14 +137,6 @@ def _sanitize_id(raw_id: str, max_len: int = 36) -> str:_SAFE_CWD_PREFIX = os.path.realpath("/tmp/copilot-")-def _encode_cwd_for_cli(cwd: str) -> str:- """Encode a working directory path the same way the Claude CLI does.-- The CLI replaces all non-alphanumeric characters with ``-``.- """- return re.sub(r"[^a-zA-Z0-9]", "-", os.path.realpath(cwd))--def cleanup_cli_project_dir(sdk_cwd: str) -> None:"""Remove the CLI's project directory for a specific working directory.@@ -188,7 +146,8 @@ def cleanup_cli_project_dir(sdk_cwd: str) -> None:"""import shutil- cwd_encoded = _encode_cwd_for_cli(sdk_cwd)+ # Encode cwd the same way CLI does (replaces non-alphanumeric with -)+ cwd_encoded = re.sub(r"[^a-zA-Z0-9]", "-", os.path.realpath(sdk_cwd))config_dir = os.environ.get("CLAUDE_CONFIG_DIR") or os.path.expanduser("~/.claude")projects_base = os.path.realpath(os.path.join(config_dir, "projects"))project_dir = os.path.realpath(os.path.join(projects_base, cwd_encoded))@@ -248,32 +207,29 @@ def write_transcript_to_tempfile(def validate_transcript(content: str | None) -> bool:"""Check that a transcript has actual conversation messages.- A valid transcript for resume needs at least one user message and one- assistant message (not just queue-operation / file-history-snapshot- metadata).+ A valid transcript needs at least one assistant message (not just+ queue-operation / file-history-snapshot metadata). We do NOT require+ a ``type: "user"`` entry because with ``--resume`` the user's message+ is passed as a CLI query parameter and does not appear in the+ transcript file."""if not content or not content.strip():return Falselines = content.strip().split("\n")- if len(lines) < 2:- return False- has_user = Falsehas_assistant = Falsefor line in lines:- try:- entry = json.loads(line)- msg_type = entry.get("type")- if msg_type == "user":- has_user = True- elif msg_type == "assistant":- has_assistant = True- except json.JSONDecodeError:+ if not line.strip():+ continue+ entry = json.loads(line, fallback=None)+ if not isinstance(entry, dict):return False+ if entry.get("type") == "assistant":+ has_assistant = True- return has_user and has_assistant+ return has_assistant# ---------------------------------------------------------------------------@@ -328,26 +284,41 @@ async def upload_transcript(session_id: str,content: str,message_count: int = 0,+ log_prefix: str = "[Transcript]",) -> None:- """Strip progress entries and upload transcript to bucket storage.+ """Strip progress entries and upload complete transcript.++ The transcript represents the FULL active context (atomic).+ Each upload REPLACES the previous transcript entirely.The executor holds a cluster lock per session, so concurrent uploads for- the same session cannot happen. We always overwrite — with ``--resume``- the CLI may compact old tool results, so neither byte size nor line count- is a reliable proxy for "newer".+ the same session cannot happen.Args:- message_count: ``len(session.messages)`` at upload time — used by- the next turn to detect staleness and compress only the gap.+ content: Complete JSONL transcript (from TranscriptBuilder).+ message_count: ``len(session.messages)`` at upload time."""from backend.util.workspace_storage import get_workspace_storage+ # Strip metadata entries (progress, file-history-snapshot, etc.)+ # Note: SDK-built transcripts shouldn't have these, but strip for safetystripped = strip_progress_entries(content)if not validate_transcript(stripped):+ # Log entry types for debugging — helps identify why validation failed+ entry_types: list[str] = []+ for line in stripped.strip().split("\n"):+ entry = json.loads(line, fallback={"type": "INVALID_JSON"})+ entry_types.append(entry.get("type", "?"))logger.warning(- f"[Transcript] Skipping upload — stripped content not valid "- f"for session {session_id}"+ "%s Skipping upload — stripped content not valid "+ "(types=%s, stripped_len=%d, raw_len=%d)",+ log_prefix,+ entry_types,+ len(stripped),+ len(content),)+ logger.debug("%s Raw content preview: %s", log_prefix, content[:500])+ logger.debug("%s Stripped content: %s", log_prefix, stripped[:500])returnstorage = await get_workspace_storage()@@ -373,17 +344,18 @@ async def upload_transcript(content=json.dumps(meta).encode("utf-8"),)except Exception as e:- logger.warning(f"[Transcript] Failed to write metadata for {session_id}: {e}")+ logger.warning(f"{log_prefix} Failed to write metadata: {e}")logger.info(- f"[Transcript] Uploaded {len(encoded)}B "- f"(stripped from {len(content)}B, msg_count={message_count}) "- f"for session {session_id}"+ f"{log_prefix} Uploaded {len(encoded)}B "+ f"(stripped from {len(content)}B, msg_count={message_count})")async def download_transcript(- user_id: str, session_id: str+ user_id: str,+ session_id: str,+ log_prefix: str = "[Transcript]",) -> TranscriptDownload | None:"""Download transcript and metadata from bucket storage.@@ -399,10 +371,10 @@ async def download_transcript(data = await storage.retrieve(path)content = data.decode("utf-8")except FileNotFoundError:- logger.debug(f"[Transcript] No transcript in storage for {session_id}")+ logger.debug(f"{log_prefix} No transcript in storage")return Noneexcept Exception as e:- logger.warning(f"[Transcript] Failed to download transcript: {e}")+ logger.warning(f"{log_prefix} Failed to download transcript: {e}")return None# Try to load metadata (best-effort — old transcripts won't have it)@@ -419,16 +391,13 @@ async def download_transcript(meta_path = f"local://{mwid}/{mfid}/{mfname}"meta_data = await storage.retrieve(meta_path)- meta = json.loads(meta_data.decode("utf-8"))+ meta = json.loads(meta_data.decode("utf-8"), fallback={})message_count = meta.get("message_count", 0)uploaded_at = meta.get("uploaded_at", 0.0)- except (FileNotFoundError, json.JSONDecodeError, Exception):+ except (FileNotFoundError, Exception):pass # No metadata — treat as unknown (msg_count=0 → always fill gap)- logger.info(- f"[Transcript] Downloaded {len(content)}B "- f"(msg_count={message_count}) for session {session_id}"- )+ logger.info(f"{log_prefix} Downloaded {len(content)}B (msg_count={message_count})")return TranscriptDownload(content=content,message_count=message_count,diff --git a/autogpt_platform/backend/backend/copilot/sdk/transcript_builder.py b/autogpt_platform/backend/backend/copilot/sdk/transcript_builder.pynew file mode 100644index 0000000000..17f7ffc8c6--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/sdk/transcript_builder.py@@ -0,0 +1,150 @@+"""Build complete JSONL transcript from SDK messages.++The transcript represents the FULL active context at any point in time.+Each upload REPLACES the previous transcript atomically.++Flow:+ Turn 1: Upload [msg1, msg2]+ Turn 2: Download [msg1, msg2] → Upload [msg1, msg2, msg3, msg4] (REPLACE)+ Turn 3: Download [msg1, msg2, msg3, msg4] → Upload [all messages] (REPLACE)++The transcript is never incremental - always the complete atomic state.+"""++import logging+from typing import Any+from uuid import uuid4++from pydantic import BaseModel++from backend.util import json++from .transcript import STRIPPABLE_TYPES++logger = logging.getLogger(__name__)+++class TranscriptEntry(BaseModel):+ """Single transcript entry (user or assistant turn)."""++ type: str+ uuid: str+ parentUuid: str | None+ message: dict[str, Any]+++class TranscriptBuilder:+ """Build complete JSONL transcript from SDK messages.++ This builder maintains the FULL conversation state, not incremental changes.+ The output is always the complete active context.+ """++ def __init__(self) -> None:+ self._entries: list[TranscriptEntry] = []+ self._last_uuid: str | None = None++ def load_previous(self, content: str, log_prefix: str = "[Transcript]") -> None:+ """Load complete previous transcript.++ This loads the FULL previous context. As new messages come in,+ we append to this state. The final output is the complete context+ (previous + new), not just the delta.+ """+ if not content or not content.strip():+ return++ lines = content.strip().split("\n")+ for line_num, line in enumerate(lines, 1):+ if not line.strip():+ continue++ data = json.loads(line, fallback=None)+ if data is None:+ logger.warning(+ "%s Failed to parse transcript line %d/%d",+ log_prefix,+ line_num,+ len(lines),+ )+ continue++ # Load all non-strippable entries (user/assistant/system/etc.)+ # Skip only STRIPPABLE_TYPES to match strip_progress_entries() behavior+ entry_type = data.get("type", "")+ if entry_type in STRIPPABLE_TYPES:+ continue++ entry = TranscriptEntry(+ type=data["type"],+ uuid=data.get("uuid") or str(uuid4()),+ parentUuid=data.get("parentUuid"),+ message=data.get("message", {}),+ )+ self._entries.append(entry)+ self._last_uuid = entry.uuid++ logger.info(+ "%s Loaded %d entries from previous transcript (last_uuid=%s)",+ log_prefix,+ len(self._entries),+ self._last_uuid[:12] if self._last_uuid else None,+ )++ def add_user_message(+ self, content: str | list[dict], uuid: str | None = None+ ) -> None:+ """Add user message to the complete context."""+ msg_uuid = uuid or str(uuid4())++ self._entries.append(+ TranscriptEntry(+ type="user",+ uuid=msg_uuid,+ parentUuid=self._last_uuid,+ message={"role": "user", "content": content},+ )+ )+ self._last_uuid = msg_uuid++ def add_assistant_message(+ self, content_blocks: list[dict], model: str = ""+ ) -> None:+ """Add assistant message to the complete context."""+ msg_uuid = str(uuid4())++ self._entries.append(+ TranscriptEntry(+ type="assistant",+ uuid=msg_uuid,+ parentUuid=self._last_uuid,+ message={+ "role": "assistant",+ "model": model,+ "content": content_blocks,+ },+ )+ )+ self._last_uuid = msg_uuid++ def to_jsonl(self) -> str:+ """Export complete context as JSONL.++ Returns the FULL conversation state (all entries), not incremental.+ This output REPLACES any previous transcript.+ """+ if not self._entries:+ return ""++ lines = [entry.model_dump_json(exclude_none=True) for entry in self._entries]+ return "\n".join(lines) + "\n"++ @property+ def entry_count(self) -> int:+ """Total number of entries in the complete context."""+ return len(self._entries)++ @property+ def is_empty(self) -> bool:+ """Whether this builder has any entries."""+ return len(self._entries) == 0diff --git a/autogpt_platform/backend/backend/copilot/sdk/transcript_test.py b/autogpt_platform/backend/backend/copilot/sdk/transcript_test.pyindex b4b65fd526..a6f50a339b 100644--- a/autogpt_platform/backend/backend/copilot/sdk/transcript_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/transcript_test.py@@ -1,11 +1,11 @@"""Unit tests for JSONL transcript management utilities."""-import jsonimport os+from backend.util import json+from .transcript import (STRIPPABLE_TYPES,- read_transcript_file,strip_progress_entries,validate_transcript,write_transcript_to_tempfile,@@ -38,49 +38,6 @@ PROGRESS_ENTRY = {VALID_TRANSCRIPT = _make_jsonl(METADATA_LINE, FILE_HISTORY, USER_MSG, ASST_MSG)-# --- read_transcript_file ------class TestReadTranscriptFile:- def test_returns_content_for_valid_file(self, tmp_path):- path = tmp_path / "session.jsonl"- path.write_text(VALID_TRANSCRIPT)- result = read_transcript_file(str(path))- assert result is not None- assert "user" in result-- def test_returns_none_for_missing_file(self):- assert read_transcript_file("/nonexistent/path.jsonl") is None-- def test_returns_none_for_empty_path(self):- assert read_transcript_file("") is None-- def test_returns_none_for_empty_file(self, tmp_path):- path = tmp_path / "empty.jsonl"- path.write_text("")- assert read_transcript_file(str(path)) is None-- def test_returns_none_for_metadata_only(self, tmp_path):- content = _make_jsonl(METADATA_LINE, FILE_HISTORY)- path = tmp_path / "meta.jsonl"- path.write_text(content)- assert read_transcript_file(str(path)) is None-- def test_returns_none_for_invalid_json(self, tmp_path):- path = tmp_path / "bad.jsonl"- path.write_text("not json\n{}\n{}\n")- assert read_transcript_file(str(path)) is None-- def test_no_size_limit(self, tmp_path):- """Large files are accepted — bucket storage has no size limit."""- big_content = {"type": "user", "uuid": "u9", "data": "x" * 1_000_000}- content = _make_jsonl(METADATA_LINE, FILE_HISTORY, big_content, ASST_MSG)- path = tmp_path / "big.jsonl"- path.write_text(content)- result = read_transcript_file(str(path))- assert result is not None--# --- write_transcript_to_tempfile ---@@ -155,12 +112,56 @@ class TestValidateTranscript:assert validate_transcript(content) is Falsedef test_assistant_only_no_user(self):+ """With --resume the user message is a CLI query param, not a transcript entry.+ A transcript with only assistant entries is valid."""content = _make_jsonl(METADATA_LINE, FILE_HISTORY, ASST_MSG)- assert validate_transcript(content) is False+ assert validate_transcript(content) is True++ def test_resume_transcript_without_user_entry(self):+ """Simulates a real --resume stop hook transcript: the CLI session file+ has summary + assistant entries but no user entry."""+ summary = {"type": "summary", "uuid": "s1", "text": "context..."}+ asst1 = {+ "type": "assistant",+ "uuid": "a1",+ "message": {"role": "assistant", "content": "Hello!"},+ }+ asst2 = {+ "type": "assistant",+ "uuid": "a2",+ "parentUuid": "a1",+ "message": {"role": "assistant", "content": "Sure, let me help."},+ }+ content = _make_jsonl(summary, asst1, asst2)+ assert validate_transcript(content) is True++ def test_single_assistant_entry(self):+ """A transcript with just one assistant line is valid — the CLI may+ produce short transcripts for simple responses with no tool use."""+ content = json.dumps(ASST_MSG) + "\n"+ assert validate_transcript(content) is Truedef test_invalid_json_returns_false(self):assert validate_transcript("not json\n{}\n{}\n") is False+ def test_malformed_json_after_valid_assistant_returns_false(self):+ """Validation must scan all lines - malformed JSON anywhere should fail."""+ valid_asst = json.dumps(ASST_MSG)+ malformed = "not valid json"+ content = valid_asst + "\n" + malformed + "\n"+ assert validate_transcript(content) is False++ def test_blank_lines_are_skipped(self):+ """Transcripts with blank lines should be valid if they contain assistant entries."""+ content = (+ json.dumps(USER_MSG)+ + "\n\n" # blank line+ + json.dumps(ASST_MSG)+ + "\n"+ + "\n" # another blank line+ )+ assert validate_transcript(content) is True+# --- strip_progress_entries ---@@ -253,3 +254,31 @@ class TestStripProgressEntries:assert "queue-operation" not in result_typesassert "user" in result_typesassert "assistant" in result_types++ def test_preserves_original_line_formatting(self):+ """Non-reparented entries keep their original JSON formatting."""+ # orjson produces compact JSON - test that we preserve the exact input+ # when no reparenting is needed (no re-serialization)+ original_line = json.dumps(USER_MSG)++ content = original_line + "\n" + json.dumps(ASST_MSG) + "\n"+ result = strip_progress_entries(content)+ result_lines = result.strip().split("\n")++ # Original line should be byte-identical (not re-serialized)+ assert result_lines[0] == original_line++ def test_reparented_entries_are_reserialized(self):+ """Entries whose parentUuid changes must be re-serialized."""+ progress = {"type": "progress", "uuid": "p1", "parentUuid": "u1"}+ asst = {+ "type": "assistant",+ "uuid": "a1",+ "parentUuid": "p1",+ "message": {"role": "assistant", "content": "done"},+ }+ content = _make_jsonl(USER_MSG, progress, asst)+ result = strip_progress_entries(content)+ lines = result.strip().split("\n")+ asst_entry = json.loads(lines[-1])+ assert asst_entry["parentUuid"] == "u1" # reparenteddiff --git a/autogpt_platform/backend/backend/util/json.py b/autogpt_platform/backend/backend/util/json.pyindex 43445202a6..d0d3a3121e 100644--- a/autogpt_platform/backend/backend/util/json.py+++ b/autogpt_platform/backend/backend/util/json.py@@ -72,19 +72,58 @@ def dumps(T = TypeVar("T")+# Sentinel value to detect when fallback is not provided+_NO_FALLBACK = object()+@overload-def loads(data: str | bytes, *args, target_type: Type[T], **kwargs) -> T: ...+def loads(+ data: str | bytes, *args, target_type: Type[T], fallback: T | None = None, **kwargs+) -> T:+ pass@overload-def loads(data: str | bytes, *args, **kwargs) -> Any: ...+def loads(data: str | bytes, *args, fallback: Any = None, **kwargs) -> Any:+ passdef loads(- data: str | bytes, *args, target_type: Type[T] | None = None, **kwargs+ data: str | bytes,+ *args,+ target_type: Type[T] | None = None,+ fallback: Any = _NO_FALLBACK,+ **kwargs,) -> Any:- parsed = orjson.loads(data)+ """Parse JSON with optional fallback on decode errors.++ Args:+ data: JSON string or bytes to parse+ target_type: Optional type to validate/cast result to+ fallback: Value to return on JSONDecodeError. If not provided, raises.+ **kwargs: Additional arguments (unused, for compatibility)++ Returns:+ Parsed JSON data, or fallback value if parsing fails++ Raises:+ orjson.JSONDecodeError: Only if fallback is not provided++ Examples:+ >>> loads('{"valid": "json"}')+ {'valid': 'json'}+ >>> loads('invalid json', fallback=None)+ None+ >>> loads('invalid json', fallback={})+ {}+ >>> loads('invalid json') # raises orjson.JSONDecodeError+ """+ try:+ parsed = orjson.loads(data)+ except orjson.JSONDecodeError:+ if fallback is not _NO_FALLBACK:+ return fallback+ raiseif target_type:return type_match(parsed, target_type)