Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): keep each run in an org its runner belongs to (#15982)

Nicholas Tindle63ffb889a446
diff --git a/autogpt_platform/backend/backend/api/features/library/db.py b/autogpt_platform/backend/backend/api/features/library/db.py
index 2fdcbd9ca7..f45b5acfc0 100644
--- a/autogpt_platform/backend/backend/api/features/library/db.py
+++ b/autogpt_platform/backend/backend/api/features/library/db.py
@@ -17,6 +17,7 @@ from backend.api.features.library.exceptions import (
FolderAlreadyExistsError,
FolderValidationError,
)
+from backend.api.features.orgs.db import resolve_member_tenancy
from backend.data.db import get_database_schema, transaction
from backend.data.execution import get_graph_execution
from backend.data.expert_attribution import resolve_attributable_expert
@@ -2118,8 +2119,9 @@ async def create_preset(
)
# A preset may only reference a graph the caller can access (owned, or in
# their library and submitted); get_graph() enforces that, None if not.
- # The preset then inherits the graph's org/team (resource-follows-parent),
- # resolved here so callers can't forget it.
+ # The preset then inherits the graph's org/team (resource-follows-parent)
+ # where the caller belongs to it, and their own otherwise, resolved here
+ # so callers can't forget it.
graph = await graph_db.get_graph(
preset.graph_id, preset.graph_version, user_id=user_id
)
@@ -2148,7 +2150,9 @@ async def create_preset(
) != (organization_id, team_id):
raise NotFoundError(f"Webhook #{webhook_id} not found")
else:
- organization_id, team_id = graph.organization_id, graph.team_id
+ organization_id, team_id = await resolve_member_tenancy(
+ user_id, graph.organization_id, graph.team_id
+ )
create_input = prisma.types.AgentPresetCreateInput(
userId=user_id,
diff --git a/autogpt_platform/backend/backend/api/features/library/db_test.py b/autogpt_platform/backend/backend/api/features/library/db_test.py
index 6604d7565d..fbffdd6246 100644
--- a/autogpt_platform/backend/backend/api/features/library/db_test.py
+++ b/autogpt_platform/backend/backend/api/features/library/db_test.py
@@ -1122,7 +1122,13 @@ async def test_fetch_execution_counts_uses_group_by(mocker):
@pytest.mark.asyncio
async def test_create_preset_inherits_graph_org(mocker):
"""A preset lives in the same org/team as the graph it runs
- (resource-follows-parent), regardless of the caller's active org."""
+ (resource-follows-parent), regardless of the caller's active org, when
+ the caller belongs to it."""
+ membership = mocker.patch.object(
+ db,
+ "resolve_member_tenancy",
+ new=AsyncMock(side_effect=lambda user_id, org_id, team_id: (org_id, team_id)),
+ )
created_row = prisma.models.AgentPreset(
deactivatedByExpertArchive=False,
id="preset-1",
@@ -1168,11 +1174,68 @@ async def test_create_preset_inherits_graph_org(mocker):
)
assert result.id == "preset-1"
+ membership.assert_awaited_once_with("test-user", "org-of-graph", "team-of-graph")
create_data = mock_preset_client.create.call_args.kwargs["data"]
assert create_data["organizationId"] == "org-of-graph"
assert create_data["teamId"] == "team-of-graph"
+async def test_a_preset_on_someone_elses_workflow_stays_in_the_users_org(mocker):
+ """The workflow can be in an org the user isn't in. A preset they save on
+ it goes to their own org and team."""
+ mocker.patch.object(
+ db,
+ "resolve_member_tenancy",
+ new=AsyncMock(return_value=("own-org", "own-team")),
+ )
+ mocker.patch.object(
+ db.graph_db,
+ "get_graph",
+ new=AsyncMock(
+ return_value=MagicMock(organization_id="other-org", team_id="other-team")
+ ),
+ )
+ mock_preset_client = AsyncMock()
+ mock_preset_client.create.return_value = prisma.models.AgentPreset(
+ deactivatedByExpertArchive=False,
+ id="preset-3",
+ userId="test-user",
+ name="My Preset",
+ description="",
+ agentGraphId="graph-1",
+ agentGraphVersion=2,
+ isActive=True,
+ isDeleted=False,
+ visibility=prisma.enums.ResourceVisibility.PRIVATE,
+ createdAt=datetime.now(),
+ updatedAt=datetime.now(),
+ InputPresets=[],
+ )
+ mocker.patch.object(
+ prisma.models.AgentPreset, "prisma", return_value=mock_preset_client
+ )
+
+ await db.create_preset(
+ user_id="test-user",
+ preset=library_model.LibraryAgentPresetCreatable(
+ inputs={},
+ credentials={},
+ graph_id="graph-1",
+ graph_version=2,
+ name="My Preset",
+ description="",
+ is_active=True,
+ ),
+ )
+
+ create_data = mock_preset_client.create.call_args.kwargs["data"]
+ assert (create_data["organizationId"], create_data["teamId"]) == (
+ "own-org",
+ "own-team",
+ )
+
+
@pytest.mark.asyncio
async def test_create_preset_tenantless_graph_creates_untagged(mocker):
"""Graphs predating org tagging have no org — the preset create must
diff --git a/autogpt_platform/backend/backend/api/features/library/triggers.py b/autogpt_platform/backend/backend/api/features/library/triggers.py
index bca0cf1d26..c68f28df67 100644
--- a/autogpt_platform/backend/backend/api/features/library/triggers.py
+++ b/autogpt_platform/backend/backend/api/features/library/triggers.py
@@ -10,6 +10,7 @@ import logging
from typing import Any
from backend.api.features.experts import experts_db
+from backend.api.features.orgs.db import resolve_member_tenancy
from backend.data.graph import get_graph
from backend.data.integrations import get_webhook
from backend.data.model import CredentialsMetaInput, GraphInput
@@ -103,7 +104,11 @@ async def setup_triggered_preset(
not_found_message=f"Expert #{expert_id} not found",
)
else:
- organization_id, team_id = graph.organization_id, graph.team_id
+ # Like a preset: the graph's org/team where the user belongs to it,
+ # their own otherwise.
+ organization_id, team_id = await resolve_member_tenancy(
+ user_id, graph.organization_id, graph.team_id
+ )
new_webhook, feedback = await setup_webhook_for_block(
user_id=user_id,
@@ -175,7 +180,9 @@ async def update_triggered_preset(
not_found_message=f"Preset #{preset_id} not found",
)
else:
- organization_id, team_id = graph.organization_id, graph.team_id
+ organization_id, team_id = await resolve_member_tenancy(
+ user_id, graph.organization_id, graph.team_id
+ )
trigger_config_with_credentials = {
**inputs,
**(
diff --git a/autogpt_platform/backend/backend/api/features/library/triggers_test.py b/autogpt_platform/backend/backend/api/features/library/triggers_test.py
index 7c23c1bdbc..91d7db47d2 100644
--- a/autogpt_platform/backend/backend/api/features/library/triggers_test.py
+++ b/autogpt_platform/backend/backend/api/features/library/triggers_test.py
@@ -51,6 +51,24 @@ def _patches(*, graph, webhook=..., feedback=None, preset=None, graph_expert=Non
)
[email protected](autouse=True)
+def _user_belongs_to_the_graphs_org():
+ """A trigger takes its graph's org/team when the user belongs to it.
+ Tests of someone else's graph patch this to give the user's own."""
+ with patch(
+ f"{_PATH}.resolve_member_tenancy",
+ new=AsyncMock(side_effect=lambda user_id, org_id, team_id: (org_id, team_id)),
+ ) as membership:
+ yield membership
+
+
+def _not_a_member():
+ return patch(
+ f"{_PATH}.resolve_member_tenancy",
+ new=AsyncMock(return_value=("own-org", "own-team")),
+ )
+
+
async def _setup(*, expert_id: str | None = None):
return await setup_triggered_preset(
user_id=_USER,
@@ -268,6 +286,27 @@ async def test_webhook_setup_rejected_raises():
create_mock.assert_not_awaited()
+async def test_a_trigger_on_someone_elses_workflow_stays_in_the_users_org():
+ """The workflow can be in an org the user isn't in. A trigger they set up
+ on it, and the runs it fires, stay in their own."""
+ p_graph, p_creds, p_webhook, p_create, p_expert = _patches(
+ graph=_graph(), preset=MagicMock(id="preset-1")
+ )
+ with (
+ _not_a_member() as membership,
+ p_graph,
+ p_creds,
+ p_create,
+ p_expert,
+ p_webhook as webhook_mock,
+ ):
+ await _setup()
+ membership.assert_awaited_once_with(_USER, "shared-org", "shared-team")
+ kwargs = webhook_mock.await_args.kwargs
+ assert (kwargs["organization_id"], kwargs["team_id"]) == ("own-org", "own-team")
+
+
# ---- update_triggered_preset ----
@@ -344,6 +383,19 @@ async def test_update_reconfigure_reregisters_and_prunes_old():
m["prune"].assert_awaited_once_with(_USER, "wh-old")
+async def test_reconfiguring_a_trigger_on_someone_elses_workflow_registers_it_in_the_users_org():
+ with _not_a_member(), _update_patches(current=_preset(webhook_id="wh-old")) as m:
+ await update_triggered_preset(
+ user_id=_USER,
+ preset_id="preset-1",
+ inputs={"repo": "owner/repo"},
+ credentials={},
+ )
+ kwargs = m["setup"].await_args.kwargs
+ assert (kwargs["organization_id"], kwargs["team_id"]) == ("own-org", "own-team")
+
+
@pytest.mark.asyncio
async def test_update_expert_trigger_keeps_personal_scope():
current = _preset(
diff --git a/autogpt_platform/backend/backend/api/features/orgs/db.py b/autogpt_platform/backend/backend/api/features/orgs/db.py
index f870135dc9..a97347747b 100644
--- a/autogpt_platform/backend/backend/api/features/orgs/db.py
+++ b/autogpt_platform/backend/backend/api/features/orgs/db.py
@@ -193,6 +193,56 @@ async def resolve_default_tenancy(user_id: str) -> tuple[str | None, str | None]
return None, None
+async def resolve_member_tenancy(
+ user_id: str, organization_id: str | None, team_id: str | None
+) -> tuple[str | None, str | None]:
+ """The org/team for something ``user_id`` creates or runs, given the ones
+ it would inherit.
+
+ A preset, a trigger or a run takes the org and team of what it comes from
+ (its workflow, preset or parent run), which needn't be an org the user is
+ in. As for a request, the pair is kept only if the user is an active
+ member of the org and, when one is given, the team, checked the way
+ ``get_request_context`` checks them, and is otherwise replaced by the
+ user's own default. Unlike a request, a team
+ the user isn't on doesn't fall back to the org's home: everyone in the org
+ sees that, and what was made for a team mustn't spread to the whole org.
+ Nothing to inherit stays ``(None, None)``.
+ """
+ if not organization_id:
+ return None, None
+ if await _is_active_member(user_id, organization_id, team_id):
+ return organization_id, team_id
+ return await resolve_default_tenancy(user_id)
+
+
+async def _is_active_member(
+ user_id: str, organization_id: str, team_id: str | None
+) -> bool:
+ org_member = await prisma.orgmember.find_unique(
+ where={"orgId_userId": {"orgId": organization_id, "userId": user_id}},
+ include={"Org": True},
+ )
+ if (
+ org_member is None
+ or org_member.status != "ACTIVE"
+ or (org_member.Org is not None and org_member.Org.deletedAt is not None)
+ ):
+ return False
+ if not team_id:
+ return True
+ team_member = await prisma.teammember.find_unique(
+ where={"teamId_userId": {"teamId": team_id, "userId": user_id}},
+ include={"Team": True},
+ )
+ return (
+ team_member is not None
+ and team_member.status == "ACTIVE"
+ and team_member.Team is not None
+ and team_member.Team.orgId == organization_id
+ )
+
+
async def _create_personal_org_for_user(
user_id: str,
slug_base: str,
diff --git a/autogpt_platform/backend/backend/api/features/orgs/db_test.py b/autogpt_platform/backend/backend/api/features/orgs/db_test.py
new file mode 100644
index 0000000000..e4e25fc497
--- /dev/null
+++ b/autogpt_platform/backend/backend/api/features/orgs/db_test.py
@@ -0,0 +1,133 @@
+"""Tests for ``resolve_member_tenancy``: which org/team something a user
+creates or runs keeps from the one it would inherit."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from backend.api.features.orgs import db as orgs_db
+
+USER = "user-1"
+
+
+def _org_member(*, status: str = "ACTIVE", deleted: bool = False):
+ return SimpleNamespace(
+ status=status,
+ Org=SimpleNamespace(deletedAt="2026-10-01" if deleted else None),
+ )
+
+
+def _team_member(*, status: str = "ACTIVE", org_id: str = "org-1"):
+ return SimpleNamespace(status=status, Team=SimpleNamespace(orgId=org_id))
+
+
+def db(mocker):
+ client = MagicMock()
+ client.orgmember.find_unique = AsyncMock(return_value=None)
+ client.teammember.find_unique = AsyncMock(return_value=None)
+ mocker.patch.object(orgs_db, "prisma", client)
+ return client
+
+
+def default_tenancy(mocker):
+ return mocker.patch.object(
+ orgs_db,
+ "resolve_default_tenancy",
+ new=AsyncMock(return_value=("own-org", "own-team")),
+ )
+
+
+async def test_a_member_keeps_the_org_and_their_team(db, default_tenancy):
+ db.orgmember.find_unique.return_value = _org_member()
+ db.teammember.find_unique.return_value = _team_member()
+
+ tenancy = await orgs_db.resolve_member_tenancy(USER, "org-1", "team-1")
+
+ assert tenancy == ("org-1", "team-1")
+ db.orgmember.find_unique.assert_awaited_once_with(
+ where={"orgId_userId": {"orgId": "org-1", "userId": USER}},
+ include={"Org": True},
+ )
+ db.teammember.find_unique.assert_awaited_once_with(
+ where={"teamId_userId": {"teamId": "team-1", "userId": USER}},
+ include={"Team": True},
+ )
+ default_tenancy.assert_not_awaited()
+
+
+async def test_someone_elses_org_is_replaced_by_the_users_own(db, default_tenancy):
+ """A user who isn't a member of the org gets their own org and team."""
+ tenancy = await orgs_db.resolve_member_tenancy(USER, "other-org", "other-team")
+
+ assert tenancy == ("own-org", "own-team")
+ default_tenancy.assert_awaited_once_with(USER)
+ db.teammember.find_unique.assert_not_awaited()
+
+
+ "membership",
+ [
+ _org_member(status="INVITED"),
+ _org_member(status="SUSPENDED"),
+ _org_member(status="REMOVED"),
+ _org_member(deleted=True),
+ ],
+ ids=["invited", "suspended", "removed", "org-deleted"],
+)
+async def test_a_membership_that_isnt_active_counts_as_none(
+ db, default_tenancy, membership
+):
+ """Even with the team's membership still in place: it counts only with
+ the org's."""
+ db.orgmember.find_unique.return_value = membership
+ db.teammember.find_unique.return_value = _team_member()
+
+ assert await orgs_db.resolve_member_tenancy(USER, "org-1", "team-1") == (
+ "own-org",
+ "own-team",
+ )
+
+
+ "team_membership",
+ [None, _team_member(status="REMOVED"), _team_member(org_id="other-org")],
+ ids=["not-on-team", "removed-from-team", "team-of-another-org"],
+)
+async def test_a_team_the_user_isnt_on_is_replaced_by_the_users_own(
+ db, default_tenancy, team_membership
+):
+ """Everyone in an org sees its home, so something made for a team doesn't
+ move there when the user isn't on that team."""
+ db.orgmember.find_unique.return_value = _org_member()
+ db.teammember.find_unique.return_value = team_membership
+
+ assert await orgs_db.resolve_member_tenancy(USER, "org-1", "team-1") == (
+ "own-org",
+ "own-team",
+ )
+ default_tenancy.assert_awaited_once_with(USER)
+
+
+async def test_a_member_given_no_team_keeps_the_org_home(db, default_tenancy):
+ db.orgmember.find_unique.return_value = _org_member()
+
+ assert await orgs_db.resolve_member_tenancy(USER, "org-1", None) == (
+ "org-1",
+ None,
+ )
+ db.teammember.find_unique.assert_not_awaited()
+
+
+async def test_nothing_to_inherit_stays_untagged(db, default_tenancy):
+ assert await orgs_db.resolve_member_tenancy(USER, None, None) == (None, None)
+ db.orgmember.find_unique.assert_not_awaited()
+ default_tenancy.assert_not_awaited()
diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.py
index bde7955226..37f05ab719 100644
--- a/autogpt_platform/backend/backend/data/db_manager.py
+++ b/autogpt_platform/backend/backend/data/db_manager.py
@@ -42,7 +42,11 @@ from backend.api.features.library.triggers import (
update_triggered_preset,
)
from backend.api.features.onboarding_dump import db as onboarding_dump_db
-from backend.api.features.orgs.db import get_user_default_team, resolve_default_tenancy
+from backend.api.features.orgs.db import (
+ get_user_default_team,
+ resolve_default_tenancy,
+ resolve_member_tenancy,
+)
from backend.api.features.search.embeddings import (
cleanup_orphaned_embeddings,
delete_content_embedding,
@@ -594,6 +598,7 @@ class DatabaseManager(AppService):
# ============ Orgs ============ #
get_user_default_team = _(get_user_default_team)
resolve_default_tenancy = _(resolve_default_tenancy)
+ resolve_member_tenancy = _(resolve_member_tenancy)
find_server_link_owner = _(platform_linking_db.find_server_link_owner)
find_user_link_owner = _(platform_linking_db.find_user_link_owner)
@@ -878,6 +883,9 @@ class DatabaseManagerClient(AppServiceClient):
get_org_credits = _(d.get_org_credits)
get_personal_org_owner = _(d.get_personal_org_owner)
+ # Orgs
+ resolve_member_tenancy = _(d.resolve_member_tenancy)
+
# Block error monitoring
get_block_error_stats = _(d.get_block_error_stats)
# Execution accuracy monitoring
@@ -1155,6 +1163,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):
find_server_link_owner = d.find_server_link_owner
get_user_default_team = d.get_user_default_team
resolve_default_tenancy = d.resolve_default_tenancy
+ resolve_member_tenancy = d.resolve_member_tenancy
find_user_link_owner = d.find_user_link_owner
resolve_server_link = d.resolve_server_link
resolve_user_link = d.resolve_user_link
diff --git a/autogpt_platform/backend/backend/executor/conftest.py b/autogpt_platform/backend/backend/executor/conftest.py
index 7593d04023..4a2ee5a12d 100644
--- a/autogpt_platform/backend/backend/executor/conftest.py
+++ b/autogpt_platform/backend/backend/executor/conftest.py
@@ -61,3 +61,20 @@ def _default_team_untenanted(mocker):
"backend.api.features.orgs.db.get_user_default_team",
new=mocker.AsyncMock(return_value=(None, None)),
)
+
+
[email protected](autouse=True)
+def _runners_belong_to_their_org(mocker):
+ """Treat the runner as a member of the org/team a run is given.
+
+ ``add_graph_execution`` keeps a run's org/team only if the user belongs to
+ it, and checks that against the database. Executor tests use synthetic
+ users and orgs, so by default the check passes the given pair through.
+ Tests of a run given someone else's org override this patch.
+ """
+ mocker.patch(
+ "backend.executor.utils.resolve_member_tenancy",
+ new=mocker.AsyncMock(
+ side_effect=lambda user_id, org_id, team_id: (org_id, team_id)
+ ),
+ )
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 5431a9904d..a294776bdb 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -32,6 +32,7 @@ from backend.data.execution import (
ExecutionStatus,
GraphExecution,
GraphExecutionEntry,
+ GraphExecutionMeta,
NodeExecutionEntry,
NodeExecutionResult,
NodesInputMasks,
@@ -143,6 +144,74 @@ def _get_execution_credit_balance(
return db_client.get_credits(graph_exec.user_id)
+RUN_OUTSIDE_ITS_ORG_ERROR = (
+ "This run was stopped because it was queued in an organization or team "
+ "you aren't a member of. New runs go to your own."
+)
+
+
+def _in_the_runners_org(
+ db_client: "DatabaseManagerClient",
+ graph_exec: GraphExecutionEntry,
+) -> bool:
+ """Whether the run is queued in an org (and team) its user belongs to.
+
+ ``add_graph_execution`` makes sure of that when it creates a run, so this
+ only fails for a run queued before it did, or one whose user has left the
+ org or team since it was created, resumed runs included. Such a run is
+ stopped rather than moved, since it may already have started there.
+ Expert runs keep their own tenancy.
+ """
+ context = graph_exec.execution_context
+ if context.expert_id or not context.organization_id:
+ return True
+ organization_id, team_id = db_client.resolve_member_tenancy(
+ graph_exec.user_id, context.organization_id, context.team_id
+ )
+ return (organization_id, team_id) == (context.organization_id, context.team_id)
+
+
+def _stop_outside_its_org(
+ db_client: "DatabaseManagerClient",
+ graph_exec: GraphExecutionEntry,
+ exec_meta: GraphExecutionMeta,
+ exec_stats: GraphExecutionStats,
+ log_metadata: LogMetadata,
+) -> None:
+ """Fail a run that failed ``_in_the_runners_org`` before it runs any
+ further, keeping the record of what it has already done.
+
+ The run's FAILED write fails its running nodes too
+ (``update_graph_execution_stats`` cascades). The ones still waiting are
+ failed here first, so a stop cut short in between leaves the run running,
+ to be picked up and stopped again. Once the run has failed, it's recorded
+ in the user's activity and analytics like any other failed run."""
+ log_metadata.warning(
+ f"Stopped graph execution #{graph_exec.graph_exec_id}: it was queued "
+ "in an org or team its user doesn't belong to."
+ )
+ for node_exec in db_client.get_node_executions(
+ graph_exec.graph_exec_id,
+ statuses=[ExecutionStatus.QUEUED, ExecutionStatus.REVIEW],
+ ):
+ update_node_execution_status(
+ db_client=db_client,
+ exec_id=node_exec.node_exec_id,
+ status=ExecutionStatus.FAILED,
+ )
+ exec_stats.error = RUN_OUTSIDE_ITS_ORG_ERROR
+ if not update_graph_execution_state(
+ db_client=db_client,
+ graph_exec_id=graph_exec.graph_exec_id,
+ status=ExecutionStatus.FAILED,
+ stats=exec_stats,
+ ):
+ return
+ exec_meta.status = ExecutionStatus.FAILED
+ activity_events.handle_run_completed(db_client, graph_exec, exec_meta, exec_stats)
+ product_analytics.handle_run_finished(graph_exec, exec_meta, exec_stats)
+
+
def _record_execution_failure(
execution_stats: GraphExecutionStats,
error: BaseException,
@@ -1091,6 +1160,12 @@ class ExecutionProcessor:
exec_stats.activity_status = None
exec_stats.correctness_score = None
+ if not _in_the_runners_org(db_client, graph_exec):
+ _stop_outside_its_org(
+ db_client, graph_exec, exec_meta, exec_stats, log_metadata
+ )
+ return
+
timing_info, status = self._on_graph_execution(
graph_exec=graph_exec,
cancel=cancel,
diff --git a/autogpt_platform/backend/backend/executor/manager_run_org_test.py b/autogpt_platform/backend/backend/executor/manager_run_org_test.py
new file mode 100644
index 0000000000..b4895cb8a2
--- /dev/null
+++ b/autogpt_platform/backend/backend/executor/manager_run_org_test.py
@@ -0,0 +1,176 @@
+"""Tests for the org check the executor makes when it picks a run up."""
+
+import threading
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from backend.data.execution import (
+ ExecutionContext,
+ ExecutionStatus,
+ GraphExecutionEntry,
+ GraphExecutionMeta,
+)
+from backend.executor.manager import (
+ RUN_OUTSIDE_ITS_ORG_ERROR,
+ ExecutionProcessor,
+ _in_the_runners_org,
+)
+from backend.util.decorator import TimingInfo
+
+
+def _run(context: ExecutionContext) -> GraphExecutionEntry:
+ return GraphExecutionEntry(
+ user_id="user-1",
+ graph_exec_id="exec-1",
+ graph_id="graph-1",
+ graph_version=1,
+ execution_context=context,
+ )
+
+
+def _run_in_another_org() -> GraphExecutionEntry:
+ return _run(ExecutionContext(organization_id="other-org", team_id="other-team"))
+
+
+ "replacement",
+ [("own-org", "own-team"), ("other-org", None), (None, None)],
+ ids=["has-an-org-of-their-own", "not-on-the-team", "no-org-to-move-to"],
+)
+def test_a_run_queued_in_an_org_or_team_its_user_isnt_in_fails_the_check(
+ replacement: tuple[str | None, str | None],
+):
+ db_client = MagicMock()
+ db_client.resolve_member_tenancy.return_value = replacement
+
+ assert not _in_the_runners_org(db_client, _run_in_another_org())
+ db_client.resolve_member_tenancy.assert_called_once_with(
+ "user-1", "other-org", "other-team"
+ )
+
+
+def test_a_run_queued_in_its_users_org_passes_the_check():
+ db_client = MagicMock()
+ db_client.resolve_member_tenancy.return_value = ("org-1", "team-1")
+
+ assert _in_the_runners_org(
+ db_client, _run(ExecutionContext(organization_id="org-1", team_id="team-1"))
+ )
+
+
+ "context",
+ [
+ ExecutionContext(organization_id="org-1", expert_id="expert-1"),
+ ExecutionContext(),
+ ],
+ ids=["expert-run", "untagged-run"],
+)
+def test_expert_and_untagged_runs_arent_checked(context: ExecutionContext):
+ db_client = MagicMock()
+
+ assert _in_the_runners_org(db_client, _run(context))
+ db_client.resolve_member_tenancy.assert_not_called()
+
+
+def _start(
+ run: GraphExecutionEntry,
+ membership: tuple[str | None, str | None],
+ status: ExecutionStatus = ExecutionStatus.QUEUED,
+ stats: GraphExecutionMeta.Stats | None = None,
+ stop_applies: bool = True,
+):
+ db_client = MagicMock()
+ db_client.get_graph_execution_meta.return_value = SimpleNamespace(
+ status=status, stats=stats, expert_id=None, trigger_source=None
+ )
+ db_client.get_node_executions.return_value = [
+ SimpleNamespace(node_exec_id="node-exec-1"),
+ SimpleNamespace(node_exec_id="node-exec-2"),
+ ]
+ db_client.resolve_member_tenancy.return_value = membership
+ processor = ExecutionProcessor()
+ processor._on_graph_execution = MagicMock(
+ return_value=(
+ TimingInfo(cpu_time=0.0, wall_time=0.0),
+ ExecutionStatus.TERMINATED,
+ )
+ )
+ with (
+ patch("backend.executor.manager.get_db_client", return_value=db_client),
+ patch("backend.executor.manager.send_execution_update"),
+ patch("backend.executor.manager.update_graph_execution_state") as state,
+ patch("backend.executor.manager.expert_posts.handle_expert_run_post"),
+ patch(
+ "backend.executor.manager.product_analytics.handle_run_finished"
+ ) as analytics,
+ ):
+ if not stop_applies:
+ state.return_value = None
+ processor.on_graph_execution(run, threading.Event(), MagicMock())
+ return processor, state, db_client, analytics
+
+
+ "status",
+ [
+ ExecutionStatus.QUEUED,
+ ExecutionStatus.INCOMPLETE,
+ ExecutionStatus.RUNNING,
+ ExecutionStatus.REVIEW,
+ ],
+)
+def test_a_run_queued_in_an_org_its_user_isnt_in_is_stopped(
+ status: ExecutionStatus,
+):
+ """A run queued before runs were checked, or whose user has since left
+ the org or team, is stopped before it runs any further, with the steps
+ still waiting in it, and its user is told why. A run picked up again
+ keeps the record of what it has already done. Like any failed run, it
+ shows in the user's activity and analytics."""
+ processor, state, db_client, analytics = _start(
+ _run_in_another_org(),
+ ("own-org", "own-team"),
+ status=status,
+ stats=GraphExecutionMeta.Stats(cost=27, node_exec_count=3),
+ )
+
+ processor._on_graph_execution.assert_not_called()
+ db_client.get_node_executions.assert_called_once_with(
+ "exec-1", statuses=[ExecutionStatus.QUEUED, ExecutionStatus.REVIEW]
+ )
+ failed_nodes = {
+ call.args[0]
+ for call in db_client.update_node_execution_status.call_args_list
+ if call.args[1] == ExecutionStatus.FAILED
+ }
+ assert failed_nodes == {"node-exec-1", "node-exec-2"}
+ assert state.call_args.kwargs["status"] == ExecutionStatus.FAILED
+ stats = state.call_args.kwargs["stats"]
+ assert stats.error == RUN_OUTSIDE_ITS_ORG_ERROR
+ assert (stats.cost, stats.node_count) == (27, 3)
+ event = db_client.create_activity_event.call_args.kwargs["draft"]
+ assert (event.event_type, event.graph_exec_id) == ("run.failed", "exec-1")
+ assert analytics.call_args.args[1].status == ExecutionStatus.FAILED
+
+
+def test_a_stop_that_doesnt_take_effect_isnt_recorded_as_a_failure():
+ """A run that has meanwhile ended some other way (its user stopped it,
+ say) can't be failed, so no failure is recorded for it."""
+ _, _, db_client, analytics = _start(
+ _run_in_another_org(), ("own-org", "own-team"), stop_applies=False
+ )
+
+ db_client.create_activity_event.assert_not_called()
+ analytics.assert_not_called()
+
+
+def test_a_run_queued_in_its_users_org_runs():
+ run = _run(ExecutionContext(organization_id="org-1", team_id="team-1"))
+
+ processor, _, _, _ = _start(run, ("org-1", "team-1"))
+
+ started = processor._on_graph_execution.call_args.kwargs["graph_exec"]
+ assert started.execution_context.organization_id == "org-1"
diff --git a/autogpt_platform/backend/backend/executor/manager_test.py b/autogpt_platform/backend/backend/executor/manager_test.py
index a0be02d7d1..b812d97332 100644
--- a/autogpt_platform/backend/backend/executor/manager_test.py
+++ b/autogpt_platform/backend/backend/executor/manager_test.py
@@ -32,6 +32,24 @@ def mock_embedding_functions():
yield
[email protected](autouse=True)
+def _runners_belong_to_their_org():
+ """Check memberships for real here, unlike the other executor tests.
+
+ These runs are picked up by the executor service, which checks a run's
+ org/team against the database in its own process, out of a patch's reach.
+ The test server puts each run in an org no one is a member of, so a run
+ kept there would be stopped; checked here too, it moves to the user's own.
+ """
+
+
[email protected](autouse=True)
+def _default_team_untenanted():
+ """Look up the user's own org and team for real: these tests use real
+ users, who have a personal org with a default team, and a run moved there
+ passes the executor service's check as it would in production."""
+
+
async def create_graph(s: SpinTestServer, g: graph.Graph, u: User) -> graph.Graph:
logger.info(f"Creating graph for user {u.id}")
return await s.agent_server.test_create_graph(CreateGraph(graph=g), u.id)
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index 18c6152a77..eed42cdeda 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -11,6 +11,7 @@ from prisma.enums import ReviewStatus
from pydantic import BaseModel, Field, JsonValue, ValidationError
from backend.api.features.experts import scheduling as experts_scheduling
+from backend.api.features.orgs.db import resolve_member_tenancy
from backend.blocks import get_block
from backend.blocks._base import Block, BlockCostType, BlockSchema, BlockType
from backend.copilot.rate_limit import UserPaywalledError, is_user_paywalled
@@ -1229,6 +1230,18 @@ async def _resolve_expert_execution_tenancy(
return await get_experts_db().resolve_private_expert_tenancy(user_id, expert_id)
+async def _resolve_runner_tenancy(
+ user_id: str, organization_id: str | None, team_id: str | None
+) -> tuple[str | None, str | None]:
+ """``resolve_member_tenancy`` for a run's org/team, called directly in the
+ API server and through the DB manager in the scheduler and executor."""
+ if prisma.is_connected():
+ return await resolve_member_tenancy(user_id, organization_id, team_id)
+ return await get_database_manager_async_client().resolve_member_tenancy(
+ user_id, organization_id, team_id
+ )
+
+
async def _enforce_expert_credential_scope(
user_id: str,
expert_id: str,
@@ -1415,6 +1428,7 @@ async def _add_graph_execution(
raise UserBannedError("This account is banned, so its workflows can't run.")
is_new_execution = graph_exec_id is None
+ tenancy_replaced = False
context_expert_id = execution_context.expert_id if execution_context else None
if expert_id is not None and context_expert_id not in (None, expert_id):
raise ValueError(
@@ -1503,6 +1517,12 @@ async def _add_graph_execution(
)
graph_exec.status = ExecutionStatus.TERMINATED
return graph_exec
+ else:
+ # Any other run resumes in the org and team it's stored in,
+ # whatever the caller passes. One whose runner has left them since
+ # isn't moved, as it has already run there: the executor stops it
+ # when it picks it up.
+ organization_id, team_id = graph_exec.organization_id, graph_exec.team_id
# The row stores the caller's raw masks and the run-time credential
# inputs separately; rebuild the credential masks the same way the
@@ -1573,6 +1593,20 @@ async def _add_graph_execution(
dry_run=dry_run,
)
+ if organization_id and not expert_id:
+ # The org/team comes from whatever started the run: a request's
+ # verified context, a parent run, a preset, a trigger or a
+ # schedule. As for a request, it's kept only if the user belongs
+ # to it; otherwise the run gets the user's own default.
+ member_org_id, member_team_id = await _resolve_runner_tenancy(
+ user_id, organization_id, team_id
+ )
+ tenancy_replaced = (member_org_id, member_team_id) != (
+ organization_id,
+ team_id,
+ )
+ organization_id, team_id = member_org_id, member_team_id
+
# Tenant a NEW execution at creation: several callers arrive with a
# falsy organization_id — legacy schedules (empty organizationId),
# sub-graphs inheriting an untenanted parent's ExecutionContext (see
@@ -1704,10 +1738,23 @@ async def _add_graph_execution(
"expert_id": expert_id,
}
)
+ elif tenancy_replaced:
+ # Billing reads the context, not the row, and a sub-graph is handed
+ # its parent's: the org/team the run was moved to must replace it.
+ execution_context = execution_context.model_copy(
+ update={"organization_id": organization_id, "team_id": team_id}
+ )
+ elif not is_new_execution:
+ # A resumed run's context gets the org/team the run is stored with,
+ # in place of any the caller's names. Review resume builds one with
+ # neither, before they're known.
+ execution_context = execution_context.model_copy(
+ update={"organization_id": organization_id, "team_id": team_id}
+ )
elif execution_context.organization_id is None and graph_exec.organization_id:
- # A caller-supplied context (e.g. review-resume, admin-requeue) may
- # be built before org/team are known. Backfill from the persisted
- # row so billing and sub-graph runs aren't tenant-blind on resume.
+ # A caller-supplied context (e.g. a sub-graph's, whose parent run is
+ # untenanted) may have no org/team. Backfill from the persisted row
+ # so billing and sub-graph runs aren't tenant-blind.
execution_context = execution_context.model_copy(
update={
"organization_id": graph_exec.organization_id,
diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.py
index 7605b24ce7..25f174dcf7 100644
--- a/autogpt_platform/backend/backend/executor/utils_test.py
+++ b/autogpt_platform/backend/backend/executor/utils_test.py
@@ -2468,6 +2468,193 @@ async def test_autopilot_execution_keeps_explicit_tenancy(mocker: MockerFixture)
assert create_kwargs["team_id"] == "shared-team"
+def _runner_is_not_in_the_given_org(mocker: MockerFixture):
+ return mocker.patch(
+ "backend.executor.utils.resolve_member_tenancy",
+ new=mocker.AsyncMock(return_value=("runner-org", "runner-team")),
+ )
+
+
+async def test_a_run_given_an_org_the_user_isnt_in_goes_to_their_own(
+ mocker: MockerFixture,
+):
+ """A run can be handed an org the user isn't in, such as one a preset
+ carries from before presets were checked. The run, and the context billing
+ reads, get the user's own."""
+ mock_edb, _ = _mock_add_graph_execution_create_path(mocker)
+ membership = _runner_is_not_in_the_given_org(mocker)
+
+ await add_graph_execution(
+ graph_id="g",
+ user_id="runner",
+ organization_id="other-org",
+ team_id="other-team",
+ )
+
+ membership.assert_awaited_once_with("runner", "other-org", "other-team")
+ create_kwargs = mock_edb.create_graph_execution.call_args.kwargs
+ assert create_kwargs["organization_id"] == "runner-org"
+ assert create_kwargs["team_id"] == "runner-team"
+ context = mock_edb.create_graph_execution.return_value.to_graph_execution_entry.call_args.kwargs[
+ "execution_context"
+ ]
+ assert (context.organization_id, context.team_id) == ("runner-org", "runner-team")
+
+
+async def test_a_sub_run_moved_to_the_runners_org_is_billed_there(
+ mocker: MockerFixture,
+):
+ """A sub-graph run is handed its parent's context, and billing reads the
+ context rather than the run: when the run moves, the context moves too."""
+ from backend.data.execution import ExecutionContext
+
+ mock_edb, _ = _mock_add_graph_execution_create_path(
+ mocker, org_id="runner-org", team_id="runner-team"
+ )
+ _runner_is_not_in_the_given_org(mocker)
+ parent_context = ExecutionContext(
+ user_id="runner",
+ parent_execution_id="parent-execution",
+ organization_id="other-org",
+ team_id="other-team",
+ )
+
+ await add_graph_execution(
+ graph_id="g",
+ user_id="runner",
+ execution_context=parent_context,
+ organization_id="other-org",
+ team_id="other-team",
+ )
+
+ create_kwargs = mock_edb.create_graph_execution.call_args.kwargs
+ assert create_kwargs["organization_id"] == "runner-org"
+ context = mock_edb.create_graph_execution.return_value.to_graph_execution_entry.call_args.kwargs[
+ "execution_context"
+ ]
+ assert (context.organization_id, context.team_id) == ("runner-org", "runner-team")
+
+
+async def test_the_org_check_runs_over_rpc_where_sub_runs_start(
+ mocker: MockerFixture,
+):
+ """Sub-graph and scheduled runs start in processes without direct DB
+ access, so the membership check has to go through the DB manager there."""
+ import backend.executor.utils as executor_utils
+
+ mock_edb, _ = _mock_add_graph_execution_create_path(mocker)
+ executor_utils.prisma.is_connected.return_value = False
+ rpc = mocker.MagicMock()
+ rpc.create_graph_execution = mock_edb.create_graph_execution
+ rpc.update_graph_execution_stats = mock_edb.update_graph_execution_stats
+ rpc.update_node_execution_status_batch = mock_edb.update_node_execution_status_batch
+ rpc.get_user_by_id = executor_utils.user_db.get_user_by_id
+ rpc.get_graph_settings = executor_utils.graph_db.get_graph_settings
+ rpc.get_or_create_workspace = executor_utils.workspace_db.get_or_create_workspace
+ rpc.increment_onboarding_runs = (
+ executor_utils.onboarding_db.increment_onboarding_runs
+ )
+ rpc.resolve_member_tenancy = mocker.AsyncMock(
+ return_value=("runner-org", "runner-team")
+ )
+ mocker.patch(
+ "backend.executor.utils.get_database_manager_async_client", return_value=rpc
+ )
+
+ await add_graph_execution(
+ graph_id="g",
+ user_id="runner",
+ organization_id="other-org",
+ team_id="other-team",
+ )
+
+ rpc.resolve_member_tenancy.assert_awaited_once_with(
+ "runner", "other-org", "other-team"
+ )
+ create_kwargs = mock_edb.create_graph_execution.call_args.kwargs
+ assert create_kwargs["organization_id"] == "runner-org"
+ assert create_kwargs["team_id"] == "runner-team"
+
+
+async def test_a_new_expert_run_isnt_put_through_the_org_check(
+ mocker: MockerFixture,
+):
+ """An expert run keeps its own tenancy rule, so the check that replaces
+ an org its runner isn't in doesn't apply to it."""
+ mock_edb, _ = _mock_add_graph_execution_create_path(mocker)
+ _mock_expert_personal_tenancy(mocker)
+ membership = _runner_is_not_in_the_given_org(mocker)
+
+ await add_graph_execution(
+ graph_id="g",
+ user_id="expert-owner",
+ expert_id="expert-1",
+ organization_id="shared-org",
+ team_id="shared-team",
+ )
+
+ membership.assert_not_awaited()
+ create_kwargs = mock_edb.create_graph_execution.call_args.kwargs
+ assert (create_kwargs["organization_id"], create_kwargs["team_id"]) == (
+ "personal-org",
+ "personal-team",
+ )
+
+
+ "caller",
+ ["no-context", "context-without-an-org", "org-passed-in", "context-in-another-org"],
+)
+async def test_a_resumed_run_stays_in_the_org_it_started_in(
+ mocker: MockerFixture, caller: str
+):
+ """A run resumed after a review or an admin requeue has already run in
+ the org and team it's stored in, so it isn't moved out of them, even when
+ its runner has left them since, and whatever the caller passes: the
+ executor stops such a run when it picks it up. Review resume hands in a
+ context that has no org yet."""
+ from backend.data.execution import ExecutionContext
+
+ context = {
+ "context-without-an-org": ExecutionContext(user_id="runner"),
+ "context-in-another-org": ExecutionContext(
+ user_id="runner", organization_id="runner-org", team_id="runner-team"
+ ),
+ }.get(caller)
+ organization_id, team_id = (
+ ("runner-org", "runner-team") if caller == "org-passed-in" else (None, None)
+ )
+ _, execution_store, queue, captured = _mock_add_graph_execution_requeue_path(
+ mocker,
+ expert_id=None,
+ organization_id="other-org",
+ team_id="other-team",
+ )
+ membership = _runner_is_not_in_the_given_org(mocker)
+
+ await add_graph_execution(
+ graph_id="g",
+ user_id="runner",
+ graph_exec_id="existing-execution",
+ execution_context=context,
+ organization_id=organization_id,
+ team_id=team_id,
+ )
+
+ membership.assert_not_called()
+ context = captured["execution_context"]
+ assert (context.organization_id, context.team_id) == ("other-org", "other-team")
+ update = execution_store.update_graph_execution_stats.await_args.kwargs
+ assert update["status"] == ExecutionStatus.QUEUED
+ assert update["update_tenancy"] is False
+ queue.publish_message.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_expert_execution_rejects_unavailable_expert_before_create(
mocker: MockerFixture,