"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Background tasks: thread execution + notification queue."""
from __future__ import annotations
import json
import logging
import os
import signal
import subprocess
import threading
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from src.agent.tools import BaseTool
from src.tools._shell_safety import broad_python_kill_error
WORKDIR = Path(__file__).resolve().parents[2]
#: Task ids with a live process anywhere in this interpreter. Restart recovery
#: must not mistake a sibling manager's running task for a previous process's.
_LIVE_TASK_IDS: set[str] = set()
_LIVE_TASK_IDS_LOCK = threading.Lock()
_COMMAND_TIMEOUT_SECONDS = 300.0
_TERMINATION_GRACE_SECONDS = 2.0
_MAX_OUTPUT_CHARS = 50_000
@dataclass(frozen=True)
class BackgroundScope:
"""Which conversation or worker owns a background task.
The host that builds the tool registry supplies this. It never appears in a
model-visible tool schema: a model must not be able to claim another
session's task, or read its output, by naming an owner.
Ownership is by ``owner_id`` alone. ``attempt_id`` records which attempt
started the task so later bookkeeping can attribute it; a later attempt of
the same owner still sees tasks started by the earlier one.
"""
owner_id: str
attempt_id: Optional[str] = None
def __post_init__(self) -> None:
if not str(self.owner_id).strip():
raise ValueError("a background scope must have a non-empty owner_id")
def _start_process(command: str) -> subprocess.Popen[str]:
"""Start a shell command in a process group that can be stopped as a unit."""
kwargs: dict[str, Any] = {
"shell": True,
"cwd": WORKDIR,
"stdout": subprocess.PIPE,
"stderr": subprocess.PIPE,
"text": True,
"encoding": "utf-8",
"errors": "replace",
}
if os.name == "nt":
kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)
else:
kwargs["start_new_session"] = True
return subprocess.Popen(command, **kwargs)
def _signal_posix_process_group(process: subprocess.Popen[str], sig: int) -> None:
try:
os.killpg(process.pid, sig)
except ProcessLookupError:
pass
except PermissionError:
# macOS answers EPERM, not ESRCH, when every member of the group has
# exited but is not yet reaped (verified 2026-10-21). The group is our
# own, so nothing is left to signal; letting it raise turned a timed-out
# command into status "error" on a loaded host.
pass
def _taskkill_windows_process_tree(process: subprocess.Popen[str]) -> None:
"""Ask Windows to stop the process and every descendant."""
try:
subprocess.run(
["taskkill", "/PID", str(process.pid), "/T", "/F"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
check=False,
)
except OSError:
if process.poll() is None:
process.kill()
def _force_stop_process_tree(process: subprocess.Popen[str]) -> None:
"""Force-stop one tracked process tree without draining its pipes."""
if os.name == "nt":
_taskkill_windows_process_tree(process)
else:
# Descendants can keep the group's pipes open after its shell exits.
_signal_posix_process_group(process, signal.SIGKILL)
def _terminate_process_tree(process: subprocess.Popen[str]) -> tuple[str, str]:
"""Terminate a timed-out process tree, drain its pipes, and reap its root."""
if os.name == "nt":
_taskkill_windows_process_tree(process)
try:
return process.communicate(timeout=_TERMINATION_GRACE_SECONDS)
except subprocess.TimeoutExpired:
if process.poll() is None:
process.kill()
return process.communicate()
_signal_posix_process_group(process, signal.SIGTERM)
try:
stdout, stderr = process.communicate(timeout=_TERMINATION_GRACE_SECONDS)
except subprocess.TimeoutExpired:
_signal_posix_process_group(process, signal.SIGKILL)
return process.communicate()
# The shell can exit before a descendant that ignored SIGTERM. The group
# retains the shell's process-group id, so this final signal prevents an
# orphan even when that descendant closed its inherited output pipes.
_signal_posix_process_group(process, signal.SIGKILL)
return stdout, stderr
def _combined_output(stdout: str | None, stderr: str | None) -> str:
return ((stdout or "") + (stderr or "")).strip()[:_MAX_OUTPUT_CHARS]
class BackgroundManager:
"""Background thread execution + notification queue, addressed by owner.
One instance is a *handle*: it carries the scope its methods read and write
under. :meth:`for_scope` hands out another handle over the same task table,
so a session and its later attempts share their tasks while a different
session sees none of them.
"""
def __init__(
self,
scope: Optional[BackgroundScope] = None,
*,
on_completion: Optional[Callable[[str, dict], None]] = None,
store: Optional[Any] = None,
) -> None:
self.tasks: Dict[str, dict] = {}
self._store = store
self._persist_lock = threading.Lock()
# (owner_id, payload). Kept as pairs so the payload shape consumers and
# existing tests see stays exactly what it was; routing is internal.
self._notifications: List[tuple[str, dict]] = []
self._lock = threading.Lock()
self._condition = threading.Condition(self._lock)
self._accepting = [True]
self._on_completion = on_completion
self._scope = scope or BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}")
@property
def scope(self) -> BackgroundScope:
"""The owner this handle reads and writes under."""
return self._scope
def for_scope(
self,
scope: BackgroundScope,
*,
on_completion: Optional[Callable[[str, dict], None]] = None,
) -> "BackgroundManager":
"""Return a handle over the same task table bound to another owner."""
other = BackgroundManager(scope=scope)
other.tasks = self.tasks
other._notifications = self._notifications
other._lock = self._lock
other._condition = self._condition
other._accepting = self._accepting
other._store = self._store
other._persist_lock = self._persist_lock
other._on_completion = (
self._on_completion if on_completion is None else on_completion
)
return other
def _owns(self, task: dict) -> bool:
return task.get("owner_id") == self._scope.owner_id
def _persist(self, task_id: str) -> None:
"""Best effort: a full or unwritable runtime root must not break the run."""
if self._store is None:
return
with self._persist_lock:
with self._lock:
task = dict(self.tasks.get(task_id) or {})
if not task:
return
try:
from src.tools.background_store import record_from_task
self._store.save(record_from_task(task_id, task))
except Exception:
logging.getLogger(__name__).exception("Could not persist background task %s", task_id)
def recover_on_start(self) -> dict:
"""Reconcile records left by a previous process, once and without reruns.
A task id this interpreter is already running is left alone: a second
service constructed in the same process must not declare the first
one's live work interrupted.
"""
if self._store is None:
return {"recovered": 0, "unchanged": 0, "unreadable": []}
from src.tools.background_store import TaskRecord, reconcile_after_restart
records, unreadable = self._store.load_all()
with _LIVE_TASK_IDS_LOCK:
live = set(_LIVE_TASK_IDS)
stale = [record for record in records if record.task_id not in live]
recovered, unchanged = reconcile_after_restart(stale)
for record in recovered:
try:
self._store.save(record)
except Exception:
logging.getLogger(__name__).exception("Could not persist recovered task %s", record.task_id)
with self._lock:
restored: list[TaskRecord] = []
for record in recovered + unchanged:
if record.task_id in self.tasks:
continue
self.tasks[record.task_id] = {
"status": record.status, "command": record.command_preview,
"result": record.result_preview, "exit_code": record.exit_code,
"owner_id": record.owner_id, "attempt_id": record.attempt_id,
"process": None, "started_at": None,
"started_wall_time": record.started_at, "ended_at": record.ended_at,
"duration_seconds": record.duration_seconds,
"recovery_reason": record.recovery_reason,
"notification_pending": record.notification_pending,
}
if record.notification_pending is True:
restored.append(record)
restored.sort(key=lambda record: (
record.ended_at if record.ended_at is not None else record.updated_at,
record.task_id,
))
self._notifications[:0] = [(record.owner_id, {
"task_id": record.task_id, "status": record.status,
"command": record.command_preview[:80],
"result": (record.result_preview or "(no output)")[:500],
"exit_code": record.exit_code,
}) for record in restored]
return {"recovered": len(recovered), "unchanged": len(unchanged), "unreadable": unreadable}
def run(self, command: str) -> str:
"""Start a background task and return its task_id.
Args:
command: Shell command to execute.
Returns:
JSON string containing status and task_id.
"""
safety_error = broad_python_kill_error(command)
if safety_error:
return json.dumps(
{"status": "error", "error": safety_error},
ensure_ascii=False,
)
task_id = uuid.uuid4().hex[:8]
started_at = time.monotonic()
with self._lock:
if not self._accepting[0]:
return json.dumps({"status": "error", "error": "Background host is shutting down"})
self.tasks[task_id] = {
"status": "running",
"result": None,
"command": command,
"exit_code": None,
"process": None,
"cancel_requested": False,
"started_at": started_at,
"started_wall_time": time.time(),
"owner_id": self._scope.owner_id,
"attempt_id": self._scope.attempt_id,
"notification_pending": True,
}
with _LIVE_TASK_IDS_LOCK:
_LIVE_TASK_IDS.add(task_id)
self._persist(task_id)
threading.Thread(target=self._execute, args=(task_id, command), daemon=True).start()
return json.dumps({"status": "ok", "task_id": task_id, "message": f"Started: {command[:80]}"})
def _execute(self, task_id: str, command: str) -> None:
exit_code: int | None = None
process: subprocess.Popen[str] | None = None
output = ""
status = "error"
try:
with self._lock:
task = self.tasks.get(task_id)
skip_spawn = task is None or bool(task.get("cancel_requested"))
if not skip_spawn:
process = _start_process(command)
with self._lock:
task = self.tasks[task_id]
task["process"] = process
task["pid"] = process.pid
cancel_requested = bool(task.get("cancel_requested"))
self._persist(task_id)
if cancel_requested:
_force_stop_process_tree(process)
try:
stdout, stderr = process.communicate(timeout=_COMMAND_TIMEOUT_SECONDS)
output = _combined_output(stdout, stderr)
exit_code = process.returncode
status = "completed" if exit_code == 0 else "error"
except subprocess.TimeoutExpired:
stdout, stderr = _terminate_process_tree(process)
partial_output = _combined_output(stdout, stderr)
timeout_message = f"Timeout ({_COMMAND_TIMEOUT_SECONDS:g}s)"
output = (
f"{partial_output}\n{timeout_message}"
if partial_output
else timeout_message
)[:_MAX_OUTPUT_CHARS]
exit_code = process.returncode
status = "timeout"
except Exception as e:
output, status = str(e), "error"
if process is not None:
# Do not publish a terminal cancellation before its root is reaped.
_terminate_process_tree(process)
exit_code = process.returncode
with self._lock:
task = self.tasks.get(task_id)
if task is None:
return
if task.get("cancel_requested"):
status = "cancelled"
cancel_message = "Cancelled by request"
output = (
f"{output}\n{cancel_message}"
if output
else cancel_message
)
output = output[:_MAX_OUTPUT_CHARS]
task["status"] = status
task["result"] = output or "(no output)"
task["exit_code"] = exit_code
task["process"] = None
started_at = task.get("started_at")
if isinstance(started_at, (int, float)):
task["duration_seconds"] = round(
max(0.0, time.monotonic() - started_at),
3,
)
payload = {
"task_id": task_id, "status": status,
"command": command[:80], "result": (output or "")[:500],
"exit_code": exit_code,
}
self._notifications.append((task["owner_id"], payload))
task["ended_at"] = time.time()
task["pid"] = getattr(process, "pid", None)
self._condition.notify_all()
self._persist(task_id)
with _LIVE_TASK_IDS_LOCK:
_LIVE_TASK_IDS.discard(task_id)
if self._on_completion is not None:
try:
self._on_completion(task["owner_id"], {**payload, "attempt_id": task.get("attempt_id")})
except Exception:
logging.getLogger(__name__).exception("Could not publish background task completion")
def cancel_all(self, *, timeout: float = 0.0) -> dict:
"""Request owner-scoped cancellation and optionally wait for actual reaping."""
with self._lock:
task_ids = [tid for tid, task in self.tasks.items()
if self._owns(task) and task["status"] in {"running", "cancelling"}]
for task_id in task_ids:
self.cancel(task_id)
with self._condition:
def pending():
return [tid for tid in task_ids
if self.tasks[tid]["status"] in {"running", "cancelling"}]
self._condition.wait_for(lambda: not pending(), timeout=max(0.0, timeout))
return {"requested": len(task_ids), "pending": pending()}
def shutdown(self, *, timeout: float = 5.0) -> list[str]:
"""Close this host's shared table, not other managers in the process."""
with self._lock:
self._accepting[0] = False
owners = {task["owner_id"] for task in self.tasks.values()
if task["status"] in {"running", "cancelling"}}
deadline = time.monotonic() + max(0.0, timeout)
for owner in owners:
remaining = max(0.0, deadline - time.monotonic())
self.for_scope(BackgroundScope(owner)).cancel_all(timeout=remaining)
with self._condition:
def pending():
return [tid for tid, task in self.tasks.items()
if task["status"] in {"running", "cancelling"}]
self._condition.wait_for(lambda: not pending(), timeout=max(0.0, deadline - time.monotonic()))
return pending()
def cancel(self, task_id: str) -> str:
"""Cancel one task by its tracked process group.
Args:
task_id: Identifier returned by :meth:`run`.
Returns:
JSON string describing the cancellation request.
"""
with self._lock:
task = self.tasks.get(task_id)
# A task owned by another session is reported exactly like a task id
# that never existed, so probing cannot confirm one exists.
if task is None or not self._owns(task):
return json.dumps({
"status": "error",
"error": f"Unknown task {task_id}",
})
task_status = task["status"]
if task_status == "cancelling":
return json.dumps({
"status": "ok",
"task_id": task_id,
"task_status": "cancelling",
"message": "Cancellation already requested",
})
if task_status != "running":
return json.dumps({
"status": "error",
"error": f"Task {task_id} is already {task_status}",
"task_status": task_status,
})
process = task.get("process")
task["cancel_requested"] = True
task["status"] = "cancelling"
self._persist(task_id)
if process is not None:
_force_stop_process_tree(process)
return json.dumps({
"status": "ok",
"task_id": task_id,
"task_status": "cancelling",
"message": "Cancellation requested",
})
def check(self, task_id: Optional[str] = None) -> str:
task: dict | None = None
task_summaries: list[tuple[str, str, str]] = []
with self._lock:
if task_id:
t = self.tasks.get(task_id)
task = dict(t) if t is not None and self._owns(t) else None
else:
task_summaries = [
(tid, t["status"], t["command"])
for tid, t in self.tasks.items()
if self._owns(t)
]
if task_id:
t = task
if not t:
return json.dumps({"status": "error", "error": f"Unknown task {task_id}"})
status = t["status"]
result = t.get("result")
payload = {
"status": status,
"command": t["command"][:60],
"result": result or "(running)",
"exit_code": t.get("exit_code"),
"attempt_id": t.get("attempt_id"),
}
if t.get("recovery_reason"):
payload["recovery_reason"] = t["recovery_reason"]
if t.get("started_wall_time") is not None:
payload["started_at"] = t["started_wall_time"]
started_at = t.get("started_at")
if isinstance(started_at, (int, float)):
duration = t.get("duration_seconds")
if not isinstance(duration, (int, float)):
duration = max(0.0, time.monotonic() - started_at)
elapsed = round(duration, 1)
payload["elapsed_seconds"] = elapsed
payload["timeout_seconds"] = _COMMAND_TIMEOUT_SECONDS
if status in {"running", "cancelling"}:
remaining = max(0.0, _COMMAND_TIMEOUT_SECONDS - duration)
payload["timeout_remaining_seconds"] = round(remaining, 1)
if not result:
payload["result"] = (
f"({status} for {elapsed:g}s; automatic timeout "
f"after {_COMMAND_TIMEOUT_SECONDS:g}s; use "
"cancel_background(task_id) to stop safely)"
)
return json.dumps(payload, ensure_ascii=False)
lines = [
f"{tid}: [{status}] {command[:60]}"
for tid, status, command in task_summaries
]
return "\n".join(lines) if lines else "No background tasks."
def drain_notifications(self) -> List[dict]:
"""Return this owner's queued notifications, oldest first.
Notifications belonging to other owners stay queued for them; draining
one session can never consume another's result. Durable delivery is
acknowledged after the in-memory claim. If that write or the process
fails, the next restart can replay a notice (at-least-once delivery).
"""
acknowledged: list[str] = []
with self._lock:
mine = [payload for owner, payload in self._notifications
if owner == self._scope.owner_id]
self._notifications[:] = [(owner, payload) for owner, payload in self._notifications
if owner != self._scope.owner_id]
for payload in mine:
task_id = payload.get("task_id")
task = self.tasks.get(task_id)
if task is not None and task.get("notification_pending") is True:
task["notification_pending"] = False
acknowledged.append(task_id)
# Do not hold the task lock while taking the persistence lock: _persist
# takes them in the opposite order on completion and cancellation.
for task_id in acknowledged:
self._persist(task_id)
return mine
_STORE_MANAGERS: dict[Path, BackgroundManager] = {}
_STORE_MANAGERS_LOCK = threading.Lock()
def _manager_for_root(root: Path) -> BackgroundManager:
from src.tools.background_store import BackgroundTaskStore
key = Path(root).resolve()
with _STORE_MANAGERS_LOCK:
if key not in _STORE_MANAGERS:
manager = BackgroundManager(store=BackgroundTaskStore(key))
summary = manager.recover_on_start()
if summary["unreadable"]:
logging.getLogger(__name__).warning("Unreadable background records: %s", summary["unreadable"])
_STORE_MANAGERS[key] = manager
return _STORE_MANAGERS[key]
def init_background_store(root: Path) -> Any:
"""Initialize and reconcile one runtime root exactly once in this process."""
return _manager_for_root(root)._store
def get_background_manager(root: Path | None = None) -> BackgroundManager:
"""Return a private handle over this runtime root's durable task table."""
if root is None:
from src.config.paths import get_runtime_root
root = get_runtime_root()
return _manager_for_root(root).for_scope(BackgroundScope(owner_id=f"isolated:{uuid.uuid4().hex}"))
class BackgroundRunTool(BaseTool):
name = "background_run"
description = (
"Run a shell command in a tracked background process group. Returns "
"task_id immediately and times out after 300 seconds. On Windows the "
"shell is cmd.exe, not PowerShell. Poll with check_background and stop "
"only with cancel_background; never use taskkill/pkill/killall."
)
parameters = {"type": "object", "properties": {
"command": {"type": "string", "description": "Shell command to run in background"},
}, "required": ["command"]}
is_readonly = False
def __init__(self, background: Optional[BackgroundManager] = None) -> None:
self.background = background if background is not None else get_background_manager()
def execute(self, **kw: Any) -> str:
return self.background.run(kw["command"])
class CheckBackgroundTool(BaseTool):
name = "check_background"
description = (
"Check background task status, elapsed time, and remaining time before "
"the 300-second automatic timeout. Omit task_id to list all. To stop "
"a running task, pass its task_id to cancel_background."
)
parameters = {"type": "object", "properties": {
"task_id": {"type": "string"},
}, "required": []}
repeatable = True
def __init__(self, background: Optional[BackgroundManager] = None) -> None:
self.background = background if background is not None else get_background_manager()
def execute(self, **kw: Any) -> str:
return self.background.check(kw.get("task_id"))
class CancelBackgroundTool(BaseTool):
"""Cancel one process group created by :class:`BackgroundRunTool`."""
name = "cancel_background"
description = (
"Cancel exactly one background_run task by task_id using its tracked "
"process group/PID. Never kills processes by executable name."
)
parameters = {
"type": "object",
"properties": {
"task_id": {
"type": "string",
"description": "task_id returned by background_run",
},
},
"required": ["task_id"],
}
is_readonly = False
def __init__(self, background: Optional[BackgroundManager] = None) -> None:
self.background = background if background is not None else get_background_manager()
def execute(self, **kw: Any) -> str:
return self.background.cancel(kw["task_id"])