AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): resolve merge conflicts with dev branch
Zamil Majdy2ff036b86bf1
diff --cc autogpt_platform/backend/backend/blocks/llm.pyindex f61f6eb331,1e2ca23c37..e76f273901--- a/autogpt_platform/backend/backend/blocks/llm.py+++ b/autogpt_platform/backend/backend/blocks/llm.py@@@ -863,71 -994,60 +999,66 @@@ async def llm_callclient = anthropic.AsyncAnthropic(api_key=credentials.api_key.get_secret_value())- try:- resp = await client.messages.create(- model=llm_model.value,- system=[- {- "type": "text",- "text": sysprompt,- "cache_control": {"type": "ephemeral"},- }- ],- messages=messages,- max_tokens=max_tokens,- tools=an_tools,- timeout=600,- )-- if not resp.content:- raise ValueError("No content returned from Anthropic.")-- tool_calls = None- for content_block in resp.content:- # Antropic is different to openai, need to iterate through- # the content blocks to find the tool calls- if content_block.type == "tool_use":- if tool_calls is None:- tool_calls = []- tool_calls.append(- ToolContentBlock(- id=content_block.id,- type=content_block.type,- function=ToolCall(- name=content_block.name,- arguments=json.dumps(content_block.input),- ),- )+ resp = await client.messages.create(+ model=llm_model.value,- system=sysprompt,++ system=[++ {++ "type": "text",++ "text": sysprompt,++ "cache_control": {"type": "ephemeral"},++ }++ ],+ messages=messages,+ max_tokens=max_tokens,+ tools=an_tools,+ timeout=600,+ )++ if not resp.content:+ raise ValueError("No content returned from Anthropic.")++ tool_calls = None+ for content_block in resp.content:+ # Antropic is different to openai, need to iterate through+ # the content blocks to find the tool calls+ if content_block.type == "tool_use":+ if tool_calls is None:+ tool_calls = []+ tool_calls.append(+ ToolContentBlock(+ id=content_block.id,+ type=content_block.type,+ function=ToolCall(+ name=content_block.name,+ arguments=json.dumps(content_block.input),+ ),)-- if not tool_calls and resp.stop_reason == "tool_use":- logger.warning(- f"Tool use stop reason but no tool calls found in content. {resp}")- reasoning = None- for content_block in resp.content:- if hasattr(content_block, "type") and content_block.type == "thinking":- reasoning = content_block.thinking- break-- return LLMResponse(- raw_response=resp,- prompt=prompt,- response=(- resp.content[0].name- if isinstance(resp.content[0], anthropic.types.ToolUseBlock)- else getattr(resp.content[0], "text", "")- ),- tool_calls=tool_calls,- prompt_tokens=resp.usage.input_tokens,- completion_tokens=resp.usage.output_tokens,- reasoning=reasoning,+ if not tool_calls and resp.stop_reason == "tool_use":+ logger.warning(+ f"Tool use stop reason but no tool calls found in content. {resp}")- except anthropic.APIError as e:- error_message = f"Anthropic API error: {str(e)}"- logger.error(error_message)- raise ValueError(error_message)++ reasoning = None+ for content_block in resp.content:+ if hasattr(content_block, "type") and content_block.type == "thinking":+ reasoning = content_block.thinking+ break++ return LLMResponse(+ raw_response=resp,+ prompt=prompt,+ response=(+ resp.content[0].name+ if isinstance(resp.content[0], anthropic.types.ToolUseBlock)+ else getattr(resp.content[0], "text", "")+ ),+ tool_calls=tool_calls,+ prompt_tokens=resp.usage.input_tokens,+ completion_tokens=resp.usage.output_tokens,+ reasoning=reasoning,+ )elif provider == "groq":if tools:raise ValueError("Groq does not support tools.")diff --cc autogpt_platform/backend/backend/copilot/baseline/service.pyindex 9f1be311ea,b9134a67b7..381c283aac--- a/autogpt_platform/backend/backend/copilot/baseline/service.py+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py@@@ -39,17 -51,38 +51,39 @@@ from backend.copilot.response_model impStreamUsage,)from backend.copilot.service import (- _build_system_prompt,+ _build_cacheable_system_prompt,- _generate_session_title,_get_openai_client,+ _update_title_async,config,)from backend.copilot.token_tracking import persist_and_record_usagefrom backend.copilot.tools import execute_tool, get_available_toolsfrom backend.copilot.tracking import track_user_message+ from backend.copilot.transcript import (+ STOP_REASON_END_TURN,+ STOP_REASON_TOOL_USE,+ TranscriptDownload,+ download_transcript,+ upload_transcript,+ validate_transcript,+ )+ from backend.copilot.transcript_builder import TranscriptBuilder+from backend.data.understanding import format_understanding_for_promptfrom backend.util.exceptions import NotFoundError- from backend.util.prompt import compress_context+ from backend.util.prompt import (+ compress_context,+ estimate_token_count,+ estimate_token_count_str,+ )+ from backend.util.tool_call_loop import (+ LLMLoopResponse,+ LLMToolCall,+ ToolCallResult,+ tool_call_loop,+ )++ if TYPE_CHECKING:+ from backend.copilot.permissions import CopilotPermissionslogger = logging.getLogger(__name__)@@@ -162,6 -929,64 +930,66 @@@ async def stream_chat_completion_baselisession = await upsert_chat_session(session)+ # Select model based on the per-request mode. 'fast' downgrades to+ # the cheaper/faster model; everything else keeps the default.+ active_model = _resolve_baseline_model(mode)++ # --- E2B sandbox setup (feature parity with SDK path) ---+ e2b_sandbox = None+ e2b_api_key = config.active_e2b_api_key+ if e2b_api_key:+ try:+ from backend.copilot.tools.e2b_sandbox import get_or_create_sandbox++ e2b_sandbox = await get_or_create_sandbox(+ session_id,+ api_key=e2b_api_key,+ template=config.e2b_sandbox_template,+ timeout=config.e2b_sandbox_timeout,+ on_timeout=config.e2b_sandbox_on_timeout,+ )+ except Exception:+ logger.warning("[Baseline] E2B sandbox setup failed", exc_info=True)++ # --- Transcript support (feature parity with SDK path) ---+ transcript_builder = TranscriptBuilder()+ transcript_covers_prefix = True++ # Build system prompt only on the first turn to avoid mid-conversation+ # changes from concurrent chats updating business understanding.+ is_first_turn = len(session.messages) <= 1+ if is_first_turn:- prompt_task = _build_system_prompt(user_id, has_conversation_history=False)++ prompt_task = _build_cacheable_system_prompt(user_id)+ else:- prompt_task = _build_system_prompt(user_id=None, has_conversation_history=True)++ prompt_task = _build_cacheable_system_prompt(None)++ # Run download + prompt build concurrently — both are independent I/O+ # on the request critical path.+ if user_id and len(session.messages) > 1:- transcript_covers_prefix, (base_system_prompt, _) = await asyncio.gather(- _load_prior_transcript(- user_id=user_id,- session_id=session_id,- session_msg_count=len(session.messages),- transcript_builder=transcript_builder,- ),- prompt_task,++ transcript_covers_prefix, (base_system_prompt, understanding) = (++ await asyncio.gather(++ _load_prior_transcript(++ user_id=user_id,++ session_id=session_id,++ session_msg_count=len(session.messages),++ transcript_builder=transcript_builder,++ ),++ prompt_task,++ )+ )+ else:- base_system_prompt, _ = await prompt_task++ base_system_prompt, understanding = await prompt_task++ # Append user message to transcript.+ # Always append when the message is present and is from the user,+ # even on duplicate-suppressed retries (is_new_message=False).+ # The loaded transcript may be stale (uploaded before the previous+ # attempt stored this message), so skipping it would leave the+ # transcript without the user turn, creating a malformed+ # assistant-after-assistant structure when the LLM reply is added.+ if message and is_user_message:+ transcript_builder.append_user(content=message)+# Generate title for new sessionsif is_user_message and not session.title:user_messages = [m for m in session.messages if m.role == "user"]@@@ -195,25 -1016,90 +1019,106 @@@{"role": "system", "content": system_prompt}]for msg in messages_for_context:- if msg.role in ("user", "assistant") and msg.content:+ if msg.role == "assistant":+ entry: dict[str, Any] = {"role": "assistant"}+ if msg.content:+ entry["content"] = msg.content+ if msg.tool_calls:+ entry["tool_calls"] = msg.tool_calls+ if msg.content or msg.tool_calls:+ openai_messages.append(entry)+ elif msg.role == "tool" and msg.tool_call_id:+ openai_messages.append(+ {+ "role": "tool",+ "tool_call_id": msg.tool_call_id,+ "content": msg.content or "",+ }+ )+ elif msg.role == "user" and msg.content:openai_messages.append({"role": msg.role, "content": msg.content})- # Inject user context into the first user message on first turn++ # Inject user context into the first user message on first turn.++ # Done before attachment/URL injection so the context prefix lands at++ # the very start of the message content.+ if is_first_turn and understanding:+ user_ctx = format_understanding_for_prompt(understanding)+ injected = False+ for msg in openai_messages:+ if msg["role"] == "user":+ msg["content"] = (+ f"<user_context>\n{user_ctx}\n</user_context>\n\n{msg['content']}"+ )+ injected = True+ break+ if not injected:+ logger.warning("[Baseline] No user message found for context injection")++ # --- File attachments (feature parity with SDK path) ---+ working_dir: str | None = None+ attachment_hint = ""+ image_blocks: list[dict[str, Any]] = []+ if file_ids and user_id:+ working_dir = tempfile.mkdtemp(prefix=f"copilot-baseline-{session_id[:8]}-")+ attachment_hint, image_blocks = await _prepare_baseline_attachments(+ file_ids, user_id, session_id, working_dir+ )++ # --- URL context ---+ context_hint = ""+ if context and context.get("url"):+ url = context["url"]+ content_text = context.get("content", "")+ if content_text:+ context_hint = (+ f"\n[The user shared a URL: {url}\n" f"Content:\n{content_text[:8000]}]"+ )+ else:+ context_hint = f"\n[The user shared a URL: {url}]"++ # Append attachment + context hints and image blocks to the last user+ # message in a single reverse scan.+ extra_hint = attachment_hint + context_hint+ if extra_hint or image_blocks:+ for i in range(len(openai_messages) - 1, -1, -1):+ if openai_messages[i].get("role") == "user":+ existing = openai_messages[i].get("content", "")+ if isinstance(existing, str):+ text = existing + "\n" + extra_hint if extra_hint else existing+ if image_blocks:+ parts: list[dict[str, Any]] = [{"type": "text", "text": text}]+ for img in image_blocks:+ parts.append(+ {+ "type": "image_url",+ "image_url": {+ "url": (+ f"data:{img['source']['media_type']};"+ f"base64,{img['source']['data']}"+ )+ },+ }+ )+ openai_messages[i]["content"] = parts+ else:+ openai_messages[i]["content"] = text+ break+tools = get_available_tools()+ # --- Permission filtering ---+ if permissions is not None:+ tools = _filter_tools_by_permissions(tools, permissions)++ # Propagate execution context so tool handlers can read session-level flags.+ set_execution_context(+ user_id,+ session,+ sandbox=e2b_sandbox,+ sdk_cwd=working_dir,+ permissions=permissions,+ )+yield StreamStart(messageId=message_id, sessionId=session_id)# Propagate user/session context to Langfuse so all LLM calls withindiff --cc autogpt_platform/backend/backend/copilot/sdk/service.pyindex ed1c63163a,8d061fbdce..9e20d65572--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@@ -31,8 -33,20 +33,21 @@@ from opentelemetry import trace as otelfrom pydantic import BaseModelfrom backend.copilot.context import get_workspace_manager+ from backend.copilot.permissions import apply_tool_permissions+ from backend.copilot.rate_limit import get_user_tier+ from backend.copilot.transcript import (+ _run_compression,+ cleanup_stale_project_dirs,+ compact_transcript,+ download_transcript,+ read_compacted_entries,+ upload_transcript,+ validate_transcript,+ write_transcript_to_tempfile,+ )+ from backend.copilot.transcript_builder import TranscriptBuilderfrom backend.data.redis_client import get_redis_async+from backend.data.understanding import format_understanding_for_promptfrom backend.executor.cluster_lock import AsyncClusterLockfrom backend.util.exceptions import NotFoundErrorfrom backend.util.settings import Settings@@@ -66,11 -83,7 +84,11 @@@ from ..response_model importStreamToolOutputAvailable,StreamUsage,)-from ..service import _build_system_prompt, _is_langfuse_configured, _update_title_async+from ..service import (+ _build_cacheable_system_prompt,- _generate_session_title,+ _is_langfuse_configured,++ _update_title_async,+)from ..token_tracking import persist_and_record_usagefrom ..tools.e2b_sandbox import get_or_create_sandbox, pause_sandbox_directfrom ..tools.sandbox import WORKSPACE_PREFIX, make_session_path