Skip to content

Self-improving AI Employees for everyone

fix(security): close the unauthenticated internal socket, decouple credential scope

trohitgf287fbbfced6
diff --git a/server/constants.py b/server/constants.py
index b488f9f2..044df4dd 100644
--- a/server/constants.py
+++ b/server/constants.py
@@ -48,7 +48,6 @@ AI_AGENT_TYPES: FrozenSet[str] = frozenset(
"ai_employee",
"rlm_agent",
"claude_code_agent",
- "codex_agent",
"vertex_managed_agent",
]
)
@@ -462,3 +461,25 @@ def detect_ai_provider(node_type: str, parameters: dict = None) -> str:
if "ollama" in nt:
return "ollama"
return "openai"
+
+
+# ---------------------------------------------------------------------------
+# Identity namespaces — TWO constants, same value, NEVER aliased.
+#
+# ``user_id`` on an execution context and ``customer_id`` on a stored
+# credential happen to share the literal "owner", but they are different
+# namespaces. Credentials are stored per-installation; tenancy scopes
+# workflows and their Context/Memory. Aliasing them means a future change
+# to who owns a workflow silently re-points every OAuth token lookup, so a
+# real login would break every OAuth-backed node.
+#
+# Enforced by tests/test_identity_namespaces.py.
+# ---------------------------------------------------------------------------
+
+#: The single-owner tenancy principal. Used when auth is disabled, and as
+#: the degenerate namespace in single-owner installs.
+OWNER_PRINCIPAL_ID: str = "owner"
+
+#: Default ``customer_id`` for stored credentials. Deliberately NOT an
+#: alias of OWNER_PRINCIPAL_ID — see the note above.
+DEFAULT_CREDENTIAL_CUSTOMER_ID: str = "owner"
diff --git a/server/nodes/tool/simple_memory/_handlers.py b/server/nodes/tool/simple_memory/_handlers.py
index d3c40d61..42ae7073 100644
--- a/server/nodes/tool/simple_memory/_handlers.py
+++ b/server/nodes/tool/simple_memory/_handlers.py
@@ -43,9 +43,23 @@ def _authenticated_owner(websocket: WebSocket) -> str:
return "owner"
+def _require_external_socket(websocket: WebSocket) -> None:
+ """The internal unauthenticated worker socket may not touch Memory.
+
+ Defence in depth behind the allowlist in ``services.authz.ws_surface``:
+ the Context handlers have carried this guard since they were written,
+ and its absence here is exactly why these six handlers were reachable
+ without credentials.
+ """
+ scope = getattr(websocket, "scope", {}) or {}
+ if scope.get("path") == "/ws/internal":
+ raise NodeUserError("Memory access requires an authenticated client")
+
+
async def _resolve_store_and_scope(
data: Dict[str, Any], websocket: WebSocket
) -> tuple[MemoryToolStore, MemoryScope]:
+ _require_external_socket(websocket)
workflow_id = str(data.get("workflow_id") or "").strip()
node_id = str(
data.get("memory_node_id") or data.get("node_id") or ""
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index 85f6d44d..e7cff52e 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -1451,6 +1451,9 @@ async def handle_refresh_model_registry(data: Dict[str, Any], websocket: WebSock
from services.ws_handler_registry import get_ws_handlers
+from services.authz import resolve_internal_handler # noqa: E402
+
+
def _resolve_handler(msg_type: str):
"""Resolve a WS message_type to its handler.
@@ -1785,7 +1788,9 @@ async def websocket_internal_endpoint(websocket: WebSocket):
msg_type = data.get("type", "")
request_id = data.get("request_id")
- handler = _resolve_handler(msg_type)
+ # Deny-by-default: this socket is unauthenticated, so it may
+ # reach only the handlers the activity worker actually needs.
+ handler = resolve_internal_handler(msg_type, _resolve_handler)
if handler:
task = asyncio.create_task(_execute_handler(handler, data, websocket, msg_type, request_id))
diff --git a/server/services/agent_runtime.py b/server/services/agent_runtime.py
index 56c2e216..c18c82b6 100644
--- a/server/services/agent_runtime.py
+++ b/server/services/agent_runtime.py
@@ -186,10 +186,13 @@ def _validated_tool_args(
try:
value = spec.args_schema.model_validate(call.args or {})
- # Preserve the security boundary used by ToolNode.execute_as_tool:
- # only fields the model actually supplied may participate in the
- # invocation/config merge. Pydantic defaults are schema metadata, not
- # model-authored arguments.
+ # Only fields the model actually supplied participate in
+ # ToolNode.execute_as_tool's ``{**node_params, **tool_args}`` merge.
+ # Materializing Pydantic defaults would let schema metadata
+ # silently override operator node configuration: a node the user
+ # set to method=POST would be reset to the schema's GET default
+ # merely because the model omitted the field. Defaults are schema
+ # metadata, not model-authored arguments.
return value.model_dump(mode="json", exclude_unset=True), None
except ValidationError as exc:
return None, str(exc)
diff --git a/server/services/authz/__init__.py b/server/services/authz/__init__.py
new file mode 100644
index 00000000..ab09d84a
--- /dev/null
+++ b/server/services/authz/__init__.py
@@ -0,0 +1,15 @@
+"""Authorization: who may act, and on what.
+
+Public surface is re-exported here so callers never import submodules
+directly and the boundary stays greppable.
+"""
+
+from services.authz.ws_surface import (
+ INTERNAL_SOCKET_HANDLERS,
+ resolve_internal_handler,
+)
+
+__all__ = [
+ "INTERNAL_SOCKET_HANDLERS",
+ "resolve_internal_handler",
+]
diff --git a/server/services/authz/ws_surface.py b/server/services/authz/ws_surface.py
new file mode 100644
index 00000000..223b8cd5
--- /dev/null
+++ b/server/services/authz/ws_surface.py
@@ -0,0 +1,54 @@
+"""Which handlers the unauthenticated internal socket may reach.
+
+``/ws/internal`` exists so a Temporal activity worker can call back into
+the running backend. It is in ``PUBLIC_PATHS`` and performs no handshake,
+so it has no authenticated principal at all — yet it dispatched through
+the same registry as the authenticated socket, which meant every handler
+was reachable: ``save_workflow``, ``delete_workflow``, and all six Memory
+handlers among them.
+
+This is a deny-by-default allowlist rather than a per-handler opt-out
+because per-handler opt-out has already failed here once: the Context
+handler checked the socket path and its Memory sibling did not. An
+allowlist inverts the default, so a newly added handler is closed until
+someone edits this named constant.
+
+Splitting the handler registry in two was considered and rejected: 40+
+plugin ``__init__`` modules self-register into one registry, so a split
+forces every plugin author to classify their handler, and whichever
+registry is the default is wrong half the time — with the wrong default
+being a silent privilege grant.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+#: Everything the activity worker legitimately needs. Both execute
+#: handlers already carry their own ``/ws/internal`` identity branch.
+INTERNAL_SOCKET_HANDLERS: frozenset[str] = frozenset(
+ {
+ "execute_node",
+ "execute_ai_node",
+ "ping",
+ }
+)
+
+
+def resolve_internal_handler(
+ msg_type: str,
+ resolver: Callable[[str], Optional[Any]],
+) -> Optional[Any]:
+ """Resolve ``msg_type`` for the internal socket, or return ``None``.
+
+ A refused type returns ``None`` so the caller emits its ordinary
+ unknown-message-type response. Refusal and "no such handler" are
+ deliberately indistinguishable — a distinct error would tell a prober
+ that ``save_workflow`` exists here but is forbidden.
+ """
+ if msg_type not in INTERNAL_SOCKET_HANDLERS:
+ return None
+ return resolver(msg_type)
+
+
+__all__ = ["INTERNAL_SOCKET_HANDLERS", "resolve_internal_handler"]
diff --git a/server/services/cli_agent/service.py b/server/services/cli_agent/service.py
index 45bb4f06..5f60c9b5 100644
--- a/server/services/cli_agent/service.py
+++ b/server/services/cli_agent/service.py
@@ -601,10 +601,16 @@ class AICliService:
for tool_info in connected_tools:
tool, execution = await ai_service._build_tool_from_node(tool_info)
if tool is None or execution is None:
- raise ValueError(
- "Could not build connected tool "
- f"{tool_info.get('node_type')!r}"
+ # Skip-and-log, matching the pre-existing contract. Raising
+ # here fails the whole batch over one unbuildable node — and
+ # this path serves every CLI agent, not only Context-bound
+ # ones, so a single bad tool took down runs that never used
+ # it. A surface that ends up entirely empty is caught below.
+ logger.warning(
+ "[cli_agent] skipping unbuildable connected tool %r",
+ tool_info.get("node_type"),
)
+ continue
entry = {
**tool_info,
"_agent_tool_name": tool.name,
diff --git a/server/services/llm/messages.py b/server/services/llm/messages.py
index 8c0deb9d..a9bbdcbb 100644
--- a/server/services/llm/messages.py
+++ b/server/services/llm/messages.py
@@ -17,6 +17,24 @@ def is_valid_message_content(content: Any) -> bool:
return bool(content)
+def _has_provider_payload(message: Any) -> bool:
+ """True when provider_state carries actual replayable content.
+
+ Providers attach provider_state to every assistant message, so its
+ presence says nothing. Its ``payload.content`` is what distinguishes a
+ compaction checkpoint (must survive, has no rendered text) from an
+ empty turn (must be dropped, or it re-serializes to an empty content
+ array the API rejects).
+ """
+ state = getattr(message, "provider_state", None)
+ if not isinstance(state, dict):
+ return bool(state)
+ payload = state.get("payload")
+ if not isinstance(payload, dict):
+ return bool(payload)
+ return any(bool(value) for value in payload.values())
+
+
def filter_empty_messages(messages: Sequence) -> List:
"""Filter out messages with empty content.
@@ -37,9 +55,17 @@ def filter_empty_messages(messages: Sequence) -> List:
# blocks are inputs to the next request, not presentation content.
if role in ("ai", "assistant"):
tool_calls = getattr(m, "tool_calls", None)
- provider_state = getattr(m, "provider_state", None)
blocks = getattr(m, "blocks", None)
- if tool_calls or provider_state or blocks:
+ # The bare PRESENCE of provider_state is not enough. Anthropic
+ # attaches it to every assistant message, so keying on presence
+ # retains genuinely empty turns (a refusal, a max_tokens stop);
+ # those re-serialize as ``{"role":"assistant","content":[]}``,
+ # which the API rejects with a 400 on every subsequent run
+ # until memory is cleared — a wedge the conversation cannot
+ # recover from. What matters is whether it carries replayable
+ # payload: a compaction checkpoint lives here with no rendered
+ # text and MUST survive.
+ if tool_calls or blocks or _has_provider_payload(m):
filtered.append(m)
continue
diff --git a/server/services/plugin/base.py b/server/services/plugin/base.py
index 15812a86..1ed52e20 100644
--- a/server/services/plugin/base.py
+++ b/server/services/plugin/base.py
@@ -1120,7 +1120,15 @@ def _make_connection_factory(
node_cls: Type[BaseNode],
context: Dict[str, Any],
) -> Callable[[str], Connection]:
- user_id = context.get("user_id", "owner")
+ from constants import DEFAULT_CREDENTIAL_CUSTOMER_ID
+
+ # Credentials are scoped by ``credential_customer_id``, NOT by the
+ # tenancy principal. Reading ``user_id`` here is what made a real
+ # authenticated subject break every OAuth-backed node: tokens are
+ # stored under the installation's customer id, not the logged-in user's.
+ user_id = context.get(
+ "credential_customer_id", DEFAULT_CREDENTIAL_CUSTOMER_ID
+ )
session_id = context.get("session_id", "default")
node_id = context.get("node_id")
# Precompute credential lookup once.
diff --git a/server/services/plugin/context.py b/server/services/plugin/context.py
index 85817fc8..4b61ed43 100644
--- a/server/services/plugin/context.py
+++ b/server/services/plugin/context.py
@@ -11,6 +11,8 @@ from __future__ import annotations
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional
+from constants import DEFAULT_CREDENTIAL_CUSTOMER_ID
+
if TYPE_CHECKING:
from services.plugin.connection import Connection
@@ -29,6 +31,12 @@ class NodeContext:
session_id: str = "default"
execution_id: Optional[str] = None
user_id: str = "owner"
+ #: Namespace for stored credentials. Distinct from ``user_id``: that is
+ #: the tenancy principal, this is which installation's OAuth tokens and
+ #: API keys to read. Keeping them separate is what lets ``user_id``
+ #: become a real authenticated subject without re-pointing every
+ #: credential lookup. See constants.DEFAULT_CREDENTIAL_CUSTOMER_ID.
+ credential_customer_id: str = "owner"
workspace_dir: Optional[str] = None
# Original outputs/nodes/edges for graph-aware nodes (console, pythonExecutor, …)
outputs: Dict[str, Any] = field(default_factory=dict)
@@ -57,6 +65,12 @@ class NodeContext:
session_id=context.get("session_id", "default"),
execution_id=context.get("execution_id"),
user_id=context.get("user_id", "owner"),
+ # Falls back to the credential default, NOT to user_id — a real
+ # authenticated subject must not silently become a credential
+ # namespace nobody has stored tokens under.
+ credential_customer_id=context.get(
+ "credential_customer_id", DEFAULT_CREDENTIAL_CUSTOMER_ID
+ ),
workspace_dir=context.get("workspace_dir"),
outputs=context.get("outputs", {}) or {},
nodes=context.get("nodes", []) or [],
diff --git a/server/tests/llm/test_providers.py b/server/tests/llm/test_providers.py
index 1c7fbd8e..2b380d75 100644
--- a/server/tests/llm/test_providers.py
+++ b/server/tests/llm/test_providers.py
@@ -499,3 +499,49 @@ class TestOpenRouterProvider:
call_kwargs = provider._client.chat.completions.create.call_args[1]
assert call_kwargs["model"] == "meta-llama/llama-3-8b"
+
+
+class TestEmptyAssistantTurnFiltering:
+ """An empty assistant turn must be dropped; a compaction checkpoint
+ living in provider_state with no rendered text must survive.
+
+ Keying on the bare presence of provider_state gets this wrong in the
+ dangerous direction: Anthropic attaches it to every assistant message,
+ so an empty turn is retained and re-sent as
+ ``{"role": "assistant", "content": []}``, which the API rejects with a
+ 400 on every subsequent run until memory is cleared.
+ """
+
+ def test_empty_turn_with_provider_state_is_dropped(self):
+ from services.llm.messages import filter_empty_messages
+ from services.llm.protocol import Message
+
+ empty = Message(
+ role="assistant",
+ content="",
+ provider_state={"provider": "anthropic", "payload": {"content": []}},
+ )
+ assert filter_empty_messages([empty]) == []
+
+ def test_compaction_checkpoint_survives(self):
+ from services.llm.messages import filter_empty_messages
+ from services.llm.protocol import Message
+
+ checkpoint = Message(
+ role="assistant",
+ content="",
+ provider_state={
+ "provider": "anthropic",
+ "payload": {
+ "content": [{"type": "compaction", "content": "durable"}]
+ },
+ },
+ )
+ assert filter_empty_messages([checkpoint]) == [checkpoint]
+
+ def test_ordinary_turns_are_unaffected(self):
+ from services.llm.messages import filter_empty_messages
+ from services.llm.protocol import Message
+
+ spoke = Message(role="assistant", content="hello")
+ assert filter_empty_messages([spoke]) == [spoke]
diff --git a/server/tests/test_identity_namespaces.py b/server/tests/test_identity_namespaces.py
new file mode 100644
index 00000000..591bd0f5
--- /dev/null
+++ b/server/tests/test_identity_namespaces.py
@@ -0,0 +1,68 @@
+"""Tenancy principal and credential namespace must never be aliased.
+
+``user_id`` (who is acting) and the credential ``customer_id`` (whose
+stored tokens to read) happen to share the literal "owner". Collapsing
+them means the day ``user_id`` becomes a real authenticated subject,
+every OAuth-backed node starts looking up tokens under a customer id
+nobody has stored anything under.
+"""
+
+from __future__ import annotations
+
+import inspect
+from pathlib import Path
+
+SERVER_DIR = Path(__file__).resolve().parents[1]
+
+
+def test_the_two_constants_exist_and_are_not_aliased():
+ import constants
+
+ assert constants.OWNER_PRINCIPAL_ID == "owner"
+ assert constants.DEFAULT_CREDENTIAL_CUSTOMER_ID == "owner"
+ source = inspect.getsource(constants)
+ # Same value, but each must be its own literal. `X = Y` would mean a
+ # change to one silently moves the other.
+ assert "DEFAULT_CREDENTIAL_CUSTOMER_ID: str = OWNER_PRINCIPAL_ID" not in source
+ assert "OWNER_PRINCIPAL_ID: str = DEFAULT_CREDENTIAL_CUSTOMER_ID" not in source
+
+
+def test_node_context_keeps_them_independent():
+ from services.plugin.context import NodeContext
+
+ ctx = NodeContext.from_legacy(
+ node_id="n", node_type="t", context={"user_id": "42"}
+ )
+ assert ctx.user_id == "42"
+ # The credential namespace must NOT follow the principal.
+ assert ctx.credential_customer_id == "owner"
+
+
+def test_connection_factory_scopes_on_the_credential_namespace():
+ """The actual coupling point: this read is what broke OAuth nodes."""
+ from services.plugin import base
+
+ source = inspect.getsource(base._make_connection_factory)
+ assert "credential_customer_id" in source
+ assert 'context.get("user_id"' not in source, (
+ "the connection factory must not scope credentials by the tenancy "
+ "principal — that re-points every OAuth token lookup"
+ )
+
+
+def test_credential_modules_do_not_import_the_tenancy_principal():
+ """An import-graph guard: credential code has no business knowing
+ about the tenancy principal, and importing it invites aliasing."""
+ targets = [
+ SERVER_DIR / "services" / "auth.py",
+ SERVER_DIR / "services" / "plugin" / "connection.py",
+ SERVER_DIR / "services" / "plugin" / "credential.py",
+ SERVER_DIR / "core" / "credentials_database.py",
+ SERVER_DIR / "core" / "credential_backends.py",
+ ]
+ offenders = [
+ p.name
+ for p in targets
+ if p.exists() and "OWNER_PRINCIPAL_ID" in p.read_text(encoding="utf-8")
+ ]
+ assert not offenders, f"credential modules import the tenancy principal: {offenders}"
diff --git a/server/tests/test_internal_socket_surface.py b/server/tests/test_internal_socket_surface.py
new file mode 100644
index 00000000..4520a101
--- /dev/null
+++ b/server/tests/test_internal_socket_surface.py
@@ -0,0 +1,74 @@
+"""`/ws/internal` is unauthenticated — it must reach almost nothing.
+
+It performs no handshake and sets no principal, yet it dispatched through
+the same registry as the authenticated socket, so `save_workflow`,
+`delete_workflow` and all six Memory handlers were reachable without
+credentials.
+"""
+
+from __future__ import annotations
+
+import inspect
+
+
+def test_allowlist_is_exactly_what_the_worker_needs():
+ from services.authz import INTERNAL_SOCKET_HANDLERS
+
+ assert INTERNAL_SOCKET_HANDLERS == {
+ "execute_node",
+ "execute_ai_node",
+ "ping",
+ }, "widening this set grants unauthenticated access — justify it in review"
+
+
+def test_every_other_registered_handler_is_refused():
+ """Generated from the LIVE registry, so it cannot go stale.
+
+ This is the check that would have caught the six Memory handlers.
+ """
+ import nodes # noqa: F401 - populates the plugin handler registry
+ from routers.websocket import MESSAGE_HANDLERS, _resolve_handler
+ from services.authz import INTERNAL_SOCKET_HANDLERS, resolve_internal_handler
+ from services.ws_handler_registry import get_ws_handlers
+
+ every = set(MESSAGE_HANDLERS) | set(get_ws_handlers())
+ assert len(every) > 50, "registry looks unpopulated; the guard would be vacuous"
+
+ reachable = {
+ name for name in every
+ if resolve_internal_handler(name, _resolve_handler) is not None
+ }
+ assert reachable <= INTERNAL_SOCKET_HANDLERS
+
+ # Spot-check the ones that actually matter.
+ for dangerous in (
+ "save_workflow",
+ "delete_workflow",
+ "get_workflow",
+ "list_memory_items",
+ "clear_memory_items",
+ "remember_memory",
+ "get_agent_context",
+ ):
+ if dangerous in every:
+ assert resolve_internal_handler(dangerous, _resolve_handler) is None, (
+ f"{dangerous} is reachable on the unauthenticated socket"
+ )
+
+
+def test_refusal_is_indistinguishable_from_unknown_type():
+ """A distinct error would confirm to a prober that a handler exists."""
+ from routers.websocket import _resolve_handler
+ from services.authz import resolve_internal_handler
+
+ assert resolve_internal_handler("save_workflow", _resolve_handler) is None
+ assert resolve_internal_handler("no_such_handler_at_all", _resolve_handler) is None
+
+
+def test_internal_loop_actually_uses_the_gate():
+ """Guards against the router being refactored back to the raw resolver."""
+ import routers.websocket as ws
+
+ source = inspect.getsource(ws)
+ internal = source[source.index("WebSocket Internal") - 4000 :]
+ assert "resolve_internal_handler(msg_type, _resolve_handler)" in internal