Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Durable profile ownership. A missing owner is never replaced automatically.
The shared application database holds routing and fencing metadata only.
Browser sockets, daemon state and process IDs remain on the owning machine.
"""
from __future__ import annotations
import asyncio
import hashlib
import hmac
import json
import time
import uuid
from datetime import datetime, timezone
from typing import Any, Optional
from urllib.parse import urlsplit
from sqlalchemy import select, update
from sqlalchemy.exc import IntegrityError
from services.plugin.base import NodeUserError
RUNTIME_EPOCH = uuid.uuid4().hex
_HEARTBEATS: dict[int, asyncio.Task] = {}
_PROCESS_LOCK: Any = None
from models.browser_owners import BrowserOwner, BrowserProfileOwner, BrowserTransientRoute as BrowserTransientRoute
def settings() -> Any:
from core.container import container
return container.settings()
def replica_id() -> str:
cfg = settings()
if getattr(cfg, "distributed_mode", False) is not True:
return "local"
value = str(getattr(cfg, "browser_replica_id", "") or "")
if getattr(cfg, "distributed_mode", False) is True and not value:
raise NodeUserError("Distributed browser execution requires BROWSER_REPLICA_ID.")
return value or "local"
def owner_queue(owner_id: str) -> str:
return "browser-owner-" + hashlib.sha256(owner_id.encode()).hexdigest()[:24]
def machine_id() -> str:
import socket
return str(getattr(settings(), "browser_machine_id", "") or socket.gethostname())
def _persistent(database: Any) -> bool:
from sqlalchemy.ext.asyncio import AsyncEngine
return isinstance(getattr(database, "engine", None), AsyncEngine)
def _binding(profile: BrowserProfileOwner, owner: BrowserOwner) -> dict:
return {"profile_id": profile.profile_id, "principal_id": profile.principal_id,
"owner_id": owner.owner_id, "runtime_epoch": owner.runtime_epoch,
"task_queue": owner_queue(owner.owner_id), "base_url": owner.base_url,
"profile_task_id": profile.task_id,
"sensitive_login": profile.sensitive_login, "needs_observation": profile.needs_observation,
"challenge_required": profile.challenge_required,
"assistance": {"reason": profile.assistance_reason, "message": profile.assistance_message, "deadline": profile.assistance_deadline} if profile.assistance_reason else None,
"available": owner.heartbeat_at > time.time() - 35}
async def register_browser_owner(database: Any) -> None:
"""Register this runtime and keep its fencing lease alive.
A second process using a live replica ID is rejected. A recovered process
may advance its own epoch only after the previous heartbeat has expired.
"""
if getattr(settings(), "distributed_mode", False) is not True:
return
global _PROCESS_LOCK
identity = replica_id()
base = str(getattr(settings(), "browser_replica_url", "") or "").rstrip("/")
parsed = urlsplit(base)
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password or parsed.query or parsed.fragment:
raise NodeUserError("BROWSER_REPLICA_URL must be a trusted backend http(s) URL.")
if _PROCESS_LOCK is None:
from nodes.browser._chrome import ProfileLock
from nodes.browser._profiles import profiles_root
lock_dir = profiles_root().parent / "owners"
lock_dir.mkdir(parents=True, exist_ok=True)
lock = ProfileLock(lock_dir / (hashlib.sha256(identity.encode()).hexdigest() + ".lock"))
if not lock.acquire():
raise NodeUserError("The browser owner process is still running. Wait for its recovery; it cannot be replaced.")
_PROCESS_LOCK = lock
async with database.get_session() as session:
row = await session.get(BrowserOwner, identity, with_for_update=True)
now = time.time()
if row is not None and row.machine_id != machine_id():
raise NodeUserError("Browser owner recovery is waiting for its original machine. A different machine requires explicit operator fencing; automatic profile transfer is disabled.")
if row is not None and row.runtime_epoch != RUNTIME_EPOCH and row.heartbeat_at > now - 35:
raise NodeUserError("Another live browser runtime uses BROWSER_REPLICA_ID; choose a unique replica ID.")
if row is None:
row = BrowserOwner(owner_id=identity, runtime_epoch=RUNTIME_EPOCH, machine_id=machine_id(), base_url=base, heartbeat_at=now)
elif row.runtime_epoch != RUNTIME_EPOCH:
await session.execute(update(BrowserProfileOwner).where(BrowserProfileOwner.owner_id == identity).values(needs_observation=True))
row.runtime_epoch, row.base_url, row.heartbeat_at = RUNTIME_EPOCH, base, now
session.add(row)
await session.commit()
key = id(database)
if key not in _HEARTBEATS or _HEARTBEATS[key].done():
async def heartbeat() -> None:
while True:
await asyncio.sleep(10)
async with database.get_session() as session:
result = await session.execute(update(BrowserOwner).where(
BrowserOwner.owner_id == identity, BrowserOwner.runtime_epoch == RUNTIME_EPOCH
).values(heartbeat_at=time.time()))
await session.commit()
if result.rowcount != 1:
return
_HEARTBEATS[key] = asyncio.create_task(heartbeat(), name="browser-owner-heartbeat")
async def unregister_browser_owner() -> None:
"""Called only after every owned Chrome has stopped."""
global _PROCESS_LOCK
tasks = list(_HEARTBEATS.values())
_HEARTBEATS.clear()
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
if _PROCESS_LOCK is not None:
_PROCESS_LOCK.release()
_PROCESS_LOCK = None
async def bind_profile(database: Any, profile_id: str, principal: str) -> dict:
if getattr(settings(), "distributed_mode", False) is not True:
result = {"profile_id": profile_id, "principal_id": principal, "owner_id": "local", "runtime_epoch": RUNTIME_EPOCH,
"task_queue": None, "base_url": "", "available": True, "sensitive_login": False, "needs_observation": False}
if _persistent(database):
async with database.get_session() as session:
row = await session.get(BrowserProfileOwner, profile_id)
if row is None:
row = BrowserProfileOwner(profile_id=profile_id, principal_id=principal, owner_id="local", runtime_epoch=RUNTIME_EPOCH)
session.add(row)
try:
await session.commit()
except IntegrityError:
await session.rollback()
row = await session.get(BrowserProfileOwner, profile_id)
if row is None:
raise
elif row.runtime_epoch != RUNTIME_EPOCH:
row.runtime_epoch = RUNTIME_EPOCH
row.needs_observation = True
await session.commit()
if row.principal_id != principal:
raise NodeUserError("Browser profile access denied.")
result.update(sensitive_login=row.sensitive_login, needs_observation=row.needs_observation,
profile_task_id=row.task_id,
challenge_required=row.challenge_required,
assistance={"reason": row.assistance_reason, "message": row.assistance_message, "deadline": row.assistance_deadline} if row.assistance_reason else None)
return result
async with database.get_session() as session:
row = await session.get(BrowserProfileOwner, profile_id)
if row is not None:
if row.principal_id != principal:
raise NodeUserError("Browser profile access denied.")
owner = await session.get(BrowserOwner, row.owner_id)
if owner is None:
raise NodeUserError("Browser unavailable — waiting for its owner.")
return _binding(row, owner)
try:
async with database.get_session() as session:
# Preparing metadata may run in an orchestration worker. Only
# backend lifecycle owns runtime registration and its process lock.
# Retain an unavailable registered owner rather than transferring.
configured_owner = replica_id()
if await session.get(BrowserOwner, configured_owner) is None:
raise NodeUserError("Browser owner is not registered. Start its backend before submitting browser tasks.")
row = BrowserProfileOwner(profile_id=profile_id, principal_id=principal, owner_id=configured_owner)
session.add(row)
await session.commit()
except IntegrityError:
pass # The winner remains the permanent owner.
return await bind_profile(database, profile_id, principal)
async def routing_for_node(database: Any, workflow_id: str, node_id: str, principal: str) -> dict:
from nodes.browser._handlers import resolve_browser_node
from nodes.browser._profiles import ProfileStore
from nodes.browser.browser import BrowserParams
graph, _node = await resolve_browser_node(principal, workflow_id, node_id)
if getattr(database, "engine", None):
from services.browser_resources import validate_saved_browser_resources
await validate_saved_browser_resources(database, workflow_id, principal, graph, {})
cfg = BrowserParams.model_validate(await database.get_node_parameters(node_id) or {})
store = ProfileStore(database)
active = []
if _persistent(database):
async with database.get_session() as session:
active = (await session.execute(select(BrowserProfileOwner).where(
BrowserProfileOwner.principal_id == principal, BrowserProfileOwner.workflow_id == workflow_id,
BrowserProfileOwner.browser_node_id == node_id, BrowserProfileOwner.task_id.is_not(None)
))).scalars().all()
if len(active) > 1:
raise NodeUserError("Browser routing is ambiguous. Wait for active tasks to finish before opening this browser.")
selected_profile = active[0].profile_id if active else cfg.profile_id
if selected_profile:
profile = await store.get(principal, selected_profile)
else:
workflow = await database.get_workflow(workflow_id)
name = getattr(workflow, "name", None) or "Browser"
profile = await store.default_for_workflow(principal, workflow_id, str(name))
from services.browser_resources import assert_browser_resource_access
await assert_browser_resource_access(database, profile_id=profile.id, principal=principal,
workflow_id=workflow_id, browser_node_id=node_id)
return {**await bind_profile(database, profile.id, principal), "workflow_id": workflow_id, "node_id": node_id,
"browser_policy": {"allowed_domains": cfg.allowed_domains, "allow_private_network": cfg.allow_private_network}}
async def assert_owner(database: Any, binding: dict, principal: str, task_id: Optional[str] = None) -> dict:
current = await bind_profile(database, str(binding["profile_id"]), principal)
if current["owner_id"] != replica_id():
raise NodeUserError("Browser unavailable — waiting for its owner.")
if _persistent(database):
async with database.get_session() as session:
if getattr(settings(), "distributed_mode", False) is True:
owner = await session.get(BrowserOwner, current["owner_id"])
if owner is None or owner.runtime_epoch != RUNTIME_EPOCH:
raise NodeUserError("This browser runtime has been fenced.")
row = await session.get(BrowserProfileOwner, current["profile_id"])
if row.task_id is not None and row.task_id != task_id:
raise NodeUserError("BrowserBusy: this profile is assigned to another task.")
return current
async def assert_runtime_epoch(database: Any) -> None:
if getattr(settings(), "distributed_mode", False) is not True:
return
async with database.get_session() as session:
owner = await session.get(BrowserOwner, replica_id())
if owner is None or owner.runtime_epoch != RUNTIME_EPOCH:
raise NodeUserError("This browser runtime has been fenced.")
async def claim_browser_task(database: Any, binding: dict, principal: str, task_id: str) -> dict:
current = {**binding, **await assert_owner(database, binding, principal, task_id)}
if _persistent(database):
async with database.get_session() as session:
result = await session.execute(update(BrowserProfileOwner).where(
BrowserProfileOwner.profile_id == binding["profile_id"],
(BrowserProfileOwner.task_id.is_(None)) | (BrowserProfileOwner.task_id == task_id)
).values(task_id=task_id, workflow_id=binding.get("workflow_id"), browser_node_id=binding.get("node_id"),
updated_at=datetime.now(timezone.utc)))
await session.commit()
if result.rowcount != 1:
raise NodeUserError("BrowserBusy: this profile is assigned to another task.")
from nodes.browser._profiles import ProfileStore
from nodes.browser._runtime import get_browser_runtime
runtime = get_browser_runtime()
controller = None
try:
profile = await ProfileStore(database).get(principal, binding["profile_id"])
controller = runtime.controller_for(profile)
if controller.task_id is None and controller._op_lock.locked():
raise NodeUserError("BrowserBusy: this profile is executing another browser step.")
frozen_policy = current.get("browser_policy")
session = None
if isinstance(frozen_policy, dict) and binding.get("workflow_id") and binding.get("node_id"):
from nodes.browser._session import BrowserSession, SessionKey
from services.netpolicy import parse_allowed_domains
policy = runtime.base_policy(allow_private_network=bool(frozen_policy.get("allow_private_network")),
allowed_domains=parse_allowed_domains(frozen_policy.get("allowed_domains", "")))
session = BrowserSession(SessionKey(principal, binding["workflow_id"], binding["node_id"]), profile.id,
label="Browser AI Agent", policy=policy)
if not controller._lease_free_for(session):
raise NodeUserError("BrowserBusy: this profile is in use by another browser session.")
if controller.lease_session is not None and controller.lease_session.session_id != session.session_id:
await controller.release_lease(controller.lease_session.session_id)
controller.claim_task(task_id)
current["profile_task_id"] = task_id
controller.sensitive_login = bool(current.get("sensitive_login"))
controller.needs_observation |= bool(current.get("needs_observation"))
controller.challenge_required = bool(current.get("challenge_required"))
controller._challenge_message = str((current.get("assistance") or {}).get("message") or "Please finish the browser challenge and hand control back.")
controller.recovered_assistance = current.get("assistance")
if session is not None:
session = runtime.register_session(session)
await controller.acquire_lease(session)
except BaseException:
if controller is not None and controller.task_id == task_id:
await controller.release_task(task_id)
if _persistent(database):
async with database.get_session() as session:
await session.execute(update(BrowserProfileOwner).where(BrowserProfileOwner.profile_id == binding["profile_id"],
BrowserProfileOwner.task_id == task_id, BrowserProfileOwner.owner_id == replica_id()).values(task_id=None, workflow_id=None, browser_node_id=None))
await session.commit()
raise
return {"claimed": True, "runtime_epoch": RUNTIME_EPOCH, "binding": current}
async def persist_control(database: Any, profile_id: str, task_id: Optional[str], *, challenge: bool, needs_observation: bool, assistance: Optional[dict]) -> None:
if not _persistent(database):
return
async with database.get_session() as session:
await session.execute(update(BrowserProfileOwner).where(
BrowserProfileOwner.profile_id == profile_id, BrowserProfileOwner.owner_id == replica_id(),
BrowserProfileOwner.task_id == task_id
).values(challenge_required=challenge, needs_observation=needs_observation,
assistance_reason=(assistance or {}).get("reason"), assistance_message=(assistance or {}).get("message"),
assistance_deadline=(assistance or {}).get("deadline"), updated_at=datetime.now(timezone.utc)))
await session.commit()
async def release_browser_task(database: Any, binding: dict, task_id: str) -> dict:
await assert_runtime_epoch(database)
from nodes.browser._runtime import get_browser_runtime
controller = get_browser_runtime().controller(binding["profile_id"])
if controller is not None:
await controller.release_task(task_id)
if _persistent(database):
async with database.get_session() as session:
result = await session.execute(update(BrowserProfileOwner).where(
BrowserProfileOwner.profile_id == binding["profile_id"], BrowserProfileOwner.task_id == task_id,
BrowserProfileOwner.owner_id == replica_id()
).values(task_id=None, workflow_id=None, browser_node_id=None, updated_at=datetime.now(timezone.utc)))
await session.commit()
return {"released": result.rowcount == 1}
return {"released": True}
async def cleanup_browser_task(database: Any, binding: dict, task_id: str, *, reset: bool = False) -> dict:
if getattr(settings(), "distributed_mode", False) is True and binding["owner_id"] != replica_id():
from nodes.browser._routing import forward_command
data = {"profile_id": binding["profile_id"], "task_id": task_id}
if reset:
data["reset"] = True
return await forward_command(binding, str(binding["principal_id"]), "cleanup_task", data)
await assert_runtime_epoch(database)
if reset and _persistent(database):
async with database.get_session() as session:
current = await session.get(BrowserProfileOwner, binding["profile_id"])
if current is None or current.task_id != task_id or current.owner_id != replica_id():
return {"released": False}
from nodes.browser._runtime import get_browser_runtime
runtime = get_browser_runtime()
controller = runtime.controller(binding["profile_id"])
if controller is not None and controller.task_id not in {None, task_id}:
return {"released": False}
if reset:
# Profile startup is deliberately shared/shielded. A terminated
# Activity can stop awaiting it while Chrome is still starting; keep
# the claim until that startup settles and its daemon can be retired.
starting = getattr(runtime, "_starting", {}).get(binding["profile_id"])
if starting is not None and not starting.done():
await asyncio.shield(starting)
controller = runtime.controller(binding["profile_id"])
if controller is not None and controller.task_id not in {None, task_id}:
return {"released": False}
running = runtime.running(binding["profile_id"]) if hasattr(runtime, "running") else None
if running is not None and controller is not None and controller.task_id == task_id:
if reset:
# Termination does not execute the agent's cancellation/finally
# path. Interrupt its command before waiting for the operation
# lock, rather than waiting indefinitely for a stuck CLI reply.
controller.needs_observation = True
await running.cli.interrupt()
# Wait for an admitted step to settle before releasing its lifetime claim.
# Suspension confirms the daemon cannot finish an old command later.
async with controller._op_lock:
if not await running.cli.suspend_for_credentials():
await runtime.stop_profile(binding["profile_id"], reason="task cleanup")
if running.chrome.is_running():
raise NodeUserError("Browser cleanup is waiting for the owner to stop its runtime.")
return await release_browser_task(database, binding, task_id)
async def set_sensitive(database: Any, profile_id: str, task_id: Optional[str], enabled: bool) -> None:
await assert_runtime_epoch(database)
if not _persistent(database):
return
async with database.get_session() as session:
result = await session.execute(update(BrowserProfileOwner).where(
BrowserProfileOwner.profile_id == profile_id, BrowserProfileOwner.owner_id == replica_id(),
BrowserProfileOwner.task_id == task_id
).values(sensitive_login=enabled, updated_at=datetime.now(timezone.utc)))
await session.commit()
if result.rowcount != 1:
raise NodeUserError("The browser task no longer owns this profile.")
def sign_forward(principal: str, method: str, path: str, body: bytes) -> str:
payload = json.dumps({"principal": principal, "method": method, "path": path,
"body": hashlib.sha256(body).hexdigest(), "expires": int(time.time()) + 30}, separators=(",", ":"))
import base64
encoded = base64.urlsafe_b64encode(payload.encode()).decode()
secret = str(getattr(settings(), "browser_router_secret", "") or settings().secret_key)
signature = hmac.new(secret.encode(), encoded.encode(), hashlib.sha256).hexdigest()
return encoded + "." + signature
def verify_forward(token: str, method: str, path: str, body: bytes) -> str:
import base64
try:
encoded, signature = token.split(".", 1)
secret = str(getattr(settings(), "browser_router_secret", "") or settings().secret_key)
expected = hmac.new(secret.encode(), encoded.encode(), hashlib.sha256).hexdigest()
if not hmac.compare_digest(signature, expected):
raise ValueError()
payload = json.loads(base64.urlsafe_b64decode(encoded))
if payload["expires"] < time.time() or payload["expires"] > time.time() + 35 or payload["method"] != method or payload["path"] != path or payload["body"] != hashlib.sha256(body).hexdigest():
raise ValueError()
return str(payload["principal"])
except (ValueError, KeyError, TypeError):
raise NodeUserError("Invalid browser owner authorization.") from None