Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(voice): give the local Workbench a device cloud token for realtime voice (#2870)

Maxa9c5fde14262
diff --git a/docs/plans/voice-input-reliability.md b/docs/plans/voice-input-reliability.md
index 1d55a2888..ab726fbe9 100644
--- a/docs/plans/voice-input-reliability.md
+++ b/docs/plans/voice-input-reliability.md
@@ -34,7 +34,9 @@ raw intermediate text to the client or requiring a transcript database. If the
recording stops exactly at a segment boundary, the client sends a small
finalize-only request containing the receipts and cursor context.
-When the browser cannot obtain a cloud token, the compatibility path remains:
+The local Workbench obtains its cloud token as a device principal; see
+`docs/voice-realtime-prototype.md`. When the browser cannot obtain a cloud
+token, the compatibility path remains:
```text
getUserMedia / AudioWorklet
diff --git a/docs/voice-realtime-prototype.md b/docs/voice-realtime-prototype.md
index 7a3af6cdc..cdb5899e5 100644
--- a/docs/voice-realtime-prototype.md
+++ b/docs/voice-realtime-prototype.md
@@ -13,6 +13,20 @@ committed to the draft. Provider credentials remain server-side. Handshake,
transport, protocol, timeout, and upstream failures activate the existing HTTP
recording fallback without changing legacy or dictation endpoint contracts.
+## Cloud token
+
+The browser opens the socket with the token from `GET /api/cloud/token`. A
+remote-access browser receives a token for its signed-in avibe.bot user,
+minted through `POST /api/v1/instances/{id}/user-token`. The Workbench opened
+on this machine's own loopback origin (a local browser or the desktop shell)
+has no avibe.bot session, so a direct, same-origin loopback request receives a
+token whose principal is the paired device, minted through
+`POST /api/v1/instances/{id}/device-token`. That token grants nothing the
+device secret does not already grant. Proxied requests, LAN setup hosts,
+requests without same-origin evidence, an unpaired instance, and a refused or
+unavailable mint get `503 cloud_unavailable`, and the browser keeps the local
+HTTP relay.
+
## Reply context
When dictation starts in an existing chat, `start` may also carry `reply`: the
@@ -28,5 +42,5 @@ and Show Page dictation have no latest reply and never send it.
The `start` schema is strict, so an unknown field closes the socket. The
browser therefore sends `reply` only when the cloud token that opened the
socket declares `realtime_reply_context`. avibe.bot returns `capabilities` with
-`POST /api/v1/instances/{id}/user-token`. `GET /api/cloud/token` relays it as a
+`user-token` and `device-token`. `GET /api/cloud/token` relays it as a
bounded list of strings. A backend that predates the field declares none.
diff --git a/tests/test_remote_access_vibe_cloud.py b/tests/test_remote_access_vibe_cloud.py
index 3c721516f..e4b82182e 100644
--- a/tests/test_remote_access_vibe_cloud.py
+++ b/tests/test_remote_access_vibe_cloud.py
@@ -3171,20 +3171,179 @@ def test_cloud_token_for_request_requires_editor_role(monkeypatch) -> None:
assert called is False
-def test_cloud_token_endpoint_local_origin_without_session_degrades_to_unavailable(
+def test_mint_device_cloud_token_posts_scope_with_device_secret_header(monkeypatch) -> None:
+ config = _cloud_broker_config()
+ captured: dict = {}
+
+ def fake_json_request(url, payload, timeout=20.0, headers=None):
+ captured.update(url=url, payload=payload, headers=headers)
+ return {"access_token": "ct_device", "token_type": "Bearer", "expires_in": 3600}
+
+ monkeypatch.setattr(remote_access, "_json_request", fake_json_request)
+
+ minted = remote_access.mint_device_cloud_token(config)
+
+ assert minted == {"access_token": "ct_device", "token_type": "Bearer", "expires_in": 3600}
+ assert captured["url"] == "https://avibe.bot/api/v1/instances/inst_123/device-token"
+ assert captured["payload"] == {"scope": "asr"}
+ assert captured["headers"] == {"X-Vibe-Device-Secret": "device-secret"}
+
+
+def test_mint_device_cloud_token_returns_none_when_unpaired_or_rejected(monkeypatch) -> None:
+ calls = 0
+
+ def unpaired_request(*args, **kwargs):
+ nonlocal calls
+ calls += 1
+ return {}
+
+ monkeypatch.setattr(remote_access, "_json_request", unpaired_request)
+ assert remote_access.mint_device_cloud_token(_config()) is None
+ assert calls == 0
+
+ config = _cloud_broker_config()
+ for error in ({"error": "not_found"}, {"error": "tunnel_not_running"}):
+
+ def rejected_request(*args, error=error, **kwargs):
+ raise remote_access.BackendRequestError(409, error)
+
+ monkeypatch.setattr(remote_access, "_json_request", rejected_request)
+ assert remote_access.mint_device_cloud_token(config) is None
+
+
+def test_cloud_token_for_local_device_matches_the_remote_token_shape(monkeypatch) -> None:
+ config = _cloud_broker_config()
+ monkeypatch.setattr(
+ remote_access,
+ "_json_request",
+ lambda *a, **k: {
+ "access_token": "ct_device",
+ "expires_in": 3600,
+ "capabilities": ["realtime_reply_context", 7],
+ },
+ )
+
+ token = remote_access.cloud_token_for_local_device(config)
+
+ assert token is not None
+ assert set(token) == {"base_url", "token", "expires_at", "scope", "capabilities"}
+ assert token["base_url"] == "https://avibe.bot"
+ assert token["token"] == "ct_device"
+ assert token["scope"] == "asr"
+ assert token["expires_at"] > int(time.time())
+ assert token["capabilities"] == ["realtime_reply_context"]
+ assert remote_access.cloud_token_for_local_device(config, scope="future") is None
+
+
+def _record_device_mints(monkeypatch) -> list[str]:
+ urls: list[str] = []
+
+ def fake_json_request(url, *args, **kwargs):
+ urls.append(url)
+ return {"access_token": "ct_device", "token_type": "Bearer", "expires_in": 3600}
+
+ monkeypatch.setattr(remote_access, "_json_request", fake_json_request)
+ return urls
+
+
+ "headers",
+ [{"Referer": "http://127.0.0.1:5123/chat"}, {"Origin": "http://127.0.0.1:5123"}],
+ ids=["referer", "origin"],
+)
+def test_cloud_token_endpoint_mints_device_token_for_same_origin_loopback_browser(
monkeypatch,
tmp_path,
+ headers,
) -> None:
- """Issue #1491: a local-origin request has no avibe.bot session cookie by
- design, so the endpoint must return the documented 503 ``cloud_unavailable``
- fallback instead of the login-required signal that triggers the frontend's
- full-page redirect to ``/auth/login`` (which the local host rejects)."""
+ """The machine's own Workbench (local browser or desktop shell) has no
+ avibe.bot session; it receives a device-principal token so realtime voice
+ can reach the cloud directly."""
monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
- config = _cloud_broker_config()
- config.save()
+ _cloud_broker_config().save()
+ urls = _record_device_mints(monkeypatch)
+
+ response = ui_server.app.test_client().get(
+ "/api/cloud/token",
+ headers=headers,
+ base_url="http://127.0.0.1:5123",
+ )
+
+ assert response.status_code == 200, response.get_json()
+ payload = response.get_json()
+ assert payload["token"] == "ct_device"
+ assert payload["base_url"] == "https://avibe.bot"
+ assert payload["scope"] == "asr"
+ assert response.headers["Cache-Control"] == "no-store, private"
+ assert urls == ["https://avibe.bot/api/v1/instances/inst_123/device-token"]
+
+
+ ("headers", "base_url", "environ"),
+ [
+ ({}, "http://127.0.0.1:5123", {}),
+ ({"Origin": "https://evil.example"}, "http://127.0.0.1:5123", {}),
+ ({"Referer": "https://evil.example/page"}, "http://127.0.0.1:5123", {}),
+ ({"Origin": "http://evil.example:5123"}, "http://evil.example:5123", {}),
+ (
+ {"Origin": "http://127.0.0.1:5123", "X-Forwarded-For": "203.0.113.9"},
+ "http://127.0.0.1:5123",
+ {},
+ ),
+ (
+ {"Origin": "http://192.168.2.10:5123"},
+ "http://192.168.2.10:5123",
+ {"REMOTE_ADDR": "192.168.2.50"},
+ ),
+ ],
+ ids=["no-origin", "foreign-origin", "foreign-referer", "rebound-host", "forwarded", "lan-peer"],
+)
+def test_cloud_token_endpoint_withholds_device_token_from_other_local_callers(
+ monkeypatch,
+ tmp_path,
+ headers,
+ base_url,
+ environ,
+) -> None:
+ """Only a direct same-origin loopback browser may receive the device
+ token, so every other caller is refused before any mint. Issue #1491: none
+ of them may get the login-required signal either; a rebound host is already
+ refused by the host guard before the endpoint runs."""
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ _cloud_broker_config().save()
+ urls = _record_device_mints(monkeypatch)
+
+ response = ui_server.app.test_client().get(
+ "/api/cloud/token",
+ headers=headers,
+ base_url=base_url,
+ environ_base=environ,
+ )
+
+ payload = response.get_json()
+ assert response.status_code in {403, 503}, payload
+ assert "token" not in payload
+ assert payload["error"] in {"cloud_unavailable", "remote_access_host_mismatch"}
+ assert urls == []
+
+
+def test_cloud_token_endpoint_falls_back_when_device_mint_is_refused(
+ monkeypatch,
+ tmp_path,
+) -> None:
+ """A backend that predates ``device-token`` or refuses it (stale runtime,
+ tunnel down) keeps today's relay fallback."""
+ monkeypatch.setenv("AVIBE_HOME", str(tmp_path))
+ _cloud_broker_config().save()
+
+ def refused(*args, **kwargs):
+ raise remote_access.BackendRequestError(404, {"error": "not_found"})
+
+ monkeypatch.setattr(remote_access, "_json_request", refused)
response = ui_server.app.test_client().get(
"/api/cloud/token",
+ headers={"Origin": "http://127.0.0.1:5123"},
base_url="http://127.0.0.1:5123",
)
diff --git a/vibe/remote_access.py b/vibe/remote_access.py
index 7ea610757..ace45783c 100644
--- a/vibe/remote_access.py
+++ b/vibe/remote_access.py
@@ -1909,14 +1909,52 @@ def mint_cloud_token(
when the cloud isn't configured or the mint fails — the caller then falls
back to the local relay.
"""
- config = config or V2Config.load()
+ return _mint_instance_cloud_token(
+ config or V2Config.load(),
+ "user-token",
+ {"sub": sub, "email": email, "scope": scope},
+ timeout=timeout,
+ )
+
+
+def mint_device_cloud_token(
+ config: V2Config | None = None,
+ *,
+ scope: str = "asr",
+ timeout: float = 8.0,
+) -> dict[str, Any] | None:
+ """Exchange this instance's device secret for a short-lived cloud token
+ whose principal is the device itself rather than an avibe.bot user.
+
+ The Workbench opened on this machine's loopback origin (a local browser or
+ the desktop shell) carries no avibe.bot session, so it cannot be vouched for
+ through ``user-token``. avibe.bot grants this token nothing the device
+ secret does not already grant, under the device-secret ASR runtime gate.
+ Returns ``None`` when the cloud isn't configured or the mint fails,
+ including against a backend that predates the endpoint.
+ """
+ return _mint_instance_cloud_token(
+ config or V2Config.load(),
+ "device-token",
+ {"scope": scope},
+ timeout=timeout,
+ )
+
+
+def _mint_instance_cloud_token(
+ config: V2Config,
+ endpoint: str,
+ payload: dict[str, Any],
+ *,
+ timeout: float,
+) -> dict[str, Any] | None:
cloud = config.remote_access.vibe_cloud
if not (cloud.enabled and cloud.instance_id and cloud.instance_secret and cloud.backend_url):
return None
try:
return _json_request(
- f"{cloud.backend_url.rstrip('/')}/api/v1/instances/{cloud.instance_id}/user-token",
- {"sub": sub, "email": email, "scope": scope},
+ f"{cloud.backend_url.rstrip('/')}/api/v1/instances/{cloud.instance_id}/{endpoint}",
+ payload,
timeout=timeout,
headers={"X-Vibe-Device-Secret": cloud.instance_secret},
)
@@ -1967,7 +2005,27 @@ def cloud_token_for_authorization(
sub = str(payload.get("sub", "")).strip()
if not email or not sub:
return None
- minted = mint_cloud_token(config, sub=sub, email=email, scope=scope)
+ return _browser_cloud_token(config, mint_cloud_token(config, sub=sub, email=email, scope=scope), scope)
+
+
+def cloud_token_for_local_device(config: V2Config, *, scope: str = "asr") -> dict[str, Any] | None:
+ """Mint a device-principal Cloud ASR token for this machine's own Workbench.
+
+ Returns the same ``{base_url, token, expires_at, scope, capabilities}``
+ shape as ``cloud_token_for_authorization``. The caller owns the decision
+ that the request is a direct loopback browser on this machine; this helper
+ never sees the request.
+ """
+ if scope != "asr":
+ return None
+ return _browser_cloud_token(config, mint_device_cloud_token(config, scope=scope), scope)
+
+
+def _browser_cloud_token(
+ config: V2Config,
+ minted: Mapping[str, Any] | None,
+ scope: str,
+) -> dict[str, Any] | None:
if not minted or not minted.get("access_token"):
return None
cloud = config.remote_access.vibe_cloud
diff --git a/vibe/ui_server.py b/vibe/ui_server.py
index f3be8be65..9322f6dae 100644
--- a/vibe/ui_server.py
+++ b/vibe/ui_server.py
@@ -7808,11 +7808,13 @@ def _remote_resource_access_context():
@app.route("/api/cloud/token", methods=["GET"])
def api_cloud_token():
- """Broker a short-lived avibe.bot user token for the workbench frontend so it
+ """Broker a short-lived avibe.bot token for the workbench frontend so it
can call the cloud directly (no tunnel relay). Exempt from the auth redirect
- (like ``/api/session``) and self-checks the session: returns 503
- ``cloud_unavailable`` when there's no authenticated user / no pairing / the
- mint fails, so the frontend cleanly falls back to the local relay."""
+ (like ``/api/session``) and self-checks the caller: a remote-access browser
+ gets a token for its signed-in user, and a direct same-origin loopback
+ browser on this machine gets one whose principal is this paired device.
+ Returns 503 ``cloud_unavailable`` for any other caller, without pairing, or
+ when the mint fails, so the frontend cleanly falls back to the local relay."""
from vibe import remote_access
config = _load_remote_access_config()
@@ -7829,7 +7831,19 @@ def api_cloud_token():
# Degrade to the documented ``cloud_unavailable`` fallback instead;
# only genuine remote-access requests get the login signal.
if not _is_remote_access_request(config):
- return jsonify({"error": "cloud_unavailable"}), 503
+ # The machine's own Workbench (a local browser or the desktop
+ # shell) acts as the instance owner, so it receives a token whose
+ # principal is this paired device. Without one it has no realtime
+ # voice. Proxies, LAN setup hosts, and pages without same-origin
+ # evidence keep the relay fallback.
+ local_result = (
+ remote_access.cloud_token_for_local_device(config)
+ if _is_direct_loopback_browser_request()
+ else None
+ )
+ if local_result is None:
+ return jsonify({"error": "cloud_unavailable"}), 503
+ return _cloud_token_response(local_result)
return jsonify({"ok": False, "error": "remote_access_login_required"}), 401
resolution = remote_access.resolve_current_authorization(config, identity)
if resolution.state == "revoked":
@@ -7845,6 +7859,10 @@ def api_cloud_token():
)
if result is None:
return jsonify({"error": "cloud_unavailable"}), 503
+ return _cloud_token_response(result)
+
+
+def _cloud_token_response(result: dict[str, Any]):
response = jsonify(result)
# Bearer material must never be cached by intermediaries (Cloudflare etc.).
response.headers["Cache-Control"] = "no-store, private"