The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat: sync @-mentioned projects into remote sandboxes (confined sandbox transport, flag ON) (#10564)
Nicky Leach53bcf3897f05
diff --git a/packages/adapter-utils/src/acpx-engine/execute.test.ts b/packages/adapter-utils/src/acpx-engine/execute.test.tsindex 12ea70891d..6006e05e22 100644--- a/packages/adapter-utils/src/acpx-engine/execute.test.ts+++ b/packages/adapter-utils/src/acpx-engine/execute.test.ts@@ -2116,6 +2116,80 @@ describe("ACPX engine remote sandbox staging seam (PR 1: workspace + cwd)", () =expect(payloadEnv.PAPERCLIP_API_KEY).not.toBe("real-run-jwt");});+ it("publishes referenced-project workspace hints repointed at their staged sandbox directories", async () => {+ const { root, stateDir, localCwd, executionTarget } = await setupRemoteSandbox();+ // A referenced project with a real host tree, so the sandbox transport stages it and returns a+ // `project-<projectId>` directory for it.+ const referencedProjectDir = path.join(root, "referenced-project-a");+ await fs.mkdir(referencedProjectDir, { recursive: true });+ await fs.writeFile(path.join(referencedProjectDir, "note.txt"), "referenced", "utf8");++ // Decode the process-session LAUNCH payload — the in-sandbox process env is carried there.+ let launchPayload: Record<string, unknown> | null = null;+ (executionTarget as { runner: unknown }).runner = createLocalSandboxRunner((input) => {+ if (input.env?.PAPERCLIP_SANDBOX_EXEC_CHANNEL === "bridge") {+ const script = input.args?.[1] ?? "";+ const match = script.match(/PAPERCLIP_PROCESS_SESSION_COMMAND_B64='([^']+)'/);+ if (match) {+ launchPayload = JSON.parse(Buffer.from(match[1]!, "base64").toString("utf8")) as Record<+ string,+ unknown+ >;+ }+ }+ });++ await runExecutor(+ { agent: "custom", agentCommand: "node ./fake-acp.js", stateDir, cwd: localCwd },+ {+ authToken: "real-run-jwt",+ executionTarget,+ context: {+ taskId: "issue-1",+ wakeReason: "issue_assigned",+ paperclipWorkspace: {+ cwd: localCwd,+ realization: {+ additional: [+ {+ path: referencedProjectDir,+ projectId: "a",+ projectWorkspaceId: "ws-a",+ repoUrl: "https://example.test/a.git",+ repoRef: "main",+ },+ ],+ },+ },+ // The plural workspace-hints channel the agent reads. The referenced hint points at the+ // host path today; on a remote target the run must repoint it at the staged directory.+ paperclipWorkspaces: [+ {+ workspaceId: "ws-a",+ cwd: referencedProjectDir,+ repoUrl: "https://example.test/a.git",+ repoRef: "main",+ projectId: "a",+ },+ ],+ },+ },+ );++ const payloadEnv = ((launchPayload as Record<string, unknown> | null)?.env ?? {}) as Record<+ string,+ unknown+ >;+ const workspacesJson = payloadEnv.PAPERCLIP_WORKSPACES_JSON;+ expect(typeof workspacesJson).toBe("string");+ const hints = JSON.parse(String(workspacesJson)) as Array<Record<string, unknown>>;+ const referencedHint = hints.find((hint) => hint.projectId === "a");+ expect(referencedHint).toBeTruthy();+ // The referenced hint repoints from the host path to its staged in-sandbox directory.+ expect(String(referencedHint!.cwd)).toContain("project-a");+ expect(referencedHint!.cwd).not.toBe(referencedProjectDir);+ });+it("stops the process-session bridge when the paperclip bridge fails under concurrency", async () => {const { stateDir, localCwd, executionTarget } = await setupRemoteSandbox();// The paperclip bridge fails; the process-session bridge — started CONCURRENTLYdiff --git a/packages/adapter-utils/src/acpx-engine/execute.ts b/packages/adapter-utils/src/acpx-engine/execute.tsindex e3321c02ed..645c105834 100644--- a/packages/adapter-utils/src/acpx-engine/execute.ts+++ b/packages/adapter-utils/src/acpx-engine/execute.ts@@ -1406,6 +1406,15 @@ async function buildRuntime(input: {contentSignature: await referencedSourceContentSignature(entry.localPath),})),);+ // Referenced-project workspace hints exposed to the agent through PAPERCLIP_WORKSPACES_JSON. The+ // list joins the anchor project's alternative workspaces with the referenced (mentioned) projects.+ // On the confined sandbox lane the run repoints each referenced hint at its staged directory after+ // staging below. Empty unless run prep resolved referenced projects or alternative workspaces.+ const workspaceHints = Array.isArray(context.paperclipWorkspaces)+ ? context.paperclipWorkspaces.filter(+ (value): value is Record<string, unknown> => typeof value === "object" && value !== null,+ )+ : [];const executionTarget = readAdapterExecutionTarget({executionTarget: input.ctx.executionTarget,legacyRemoteExecution: input.ctx.executionTransport?.remoteExecution,@@ -1888,6 +1897,27 @@ async function buildRuntime(input: {remoteManagedHomeTeardown = staged.value.teardown;remoteStagingDispose = staged.value.dispose;remoteStagingEnvDelta = staged.value.envDelta;+ // Publish the referenced-project workspace hints to the in-sandbox agent. The staged-directory+ // map (`project-<projectId>`) is known only after staging above, so this runs here rather than+ // with the initial workspace shaping. Each referenced hint repoints at its staged directory; a+ // referenced hint whose project did not stage loses its cwd, so the agent never receives an+ // unstaged path. Only the confined sandbox lane stages referenced trees, so only it publishes+ // the hints; the local and runner-less lanes keep their env untouched. The set `env` write wins+ // over an inherited value in the merged launch env.+ const stagedProjectDirs = stagedRuntime?.additionalSourceDirs ?? {};+ if (Object.keys(stagedProjectDirs).length > 0) {+ const shapedHints = shapePaperclipWorkspaceEnvForExecution({+ workspaceCwd: effectiveWorkspaceCwd,+ workspaceWorktreePath,+ workspaceHints,+ executionTargetIsRemote,+ executionCwd: effectiveExecutionCwd,+ stagedProjectDirs,+ }).workspaceHints;+ if (shapedHints.length > 0) {+ env.PAPERCLIP_WORKSPACES_JSON = JSON.stringify(shapedHints);+ }+ }}// Both bridge starts run under one try so a failure at EITHER — including the// paperclip callback bridge — fires the same abandon-path cleanup. The@@ -2916,6 +2946,17 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {rootSpan.end(true);throw err;}+ // Per-project staging outcomes for the referenced (mentioned) projects, surfaced back to the+ // server on the run result. A referenced project that failed to stage into the sandbox is a+ // first-class, counted failure in the requested-vs-synced observability, not only a warning. The+ // list is empty on a local target, on a transport that does not stage referenced projects, or+ // when every staged referenced project succeeded, so the spread adds the field only when there+ // is a failure to report.+ const referencedProjectStagingFailures = (+ prepared.stagedRuntime?.additionalSourceFailures ?? []+ ).map((failure) => ({ projectId: failure.projectId }));+ const referencedProjectStagingFailuresField =+ referencedProjectStagingFailures.length > 0 ? { referencedProjectStagingFailures } : {};// State the effective wall-clock timeout and its source up front so a// later timeout is diagnosable from the run log alone. Goes to stderr:// the acpx stdout log stream carries JSON acpx.* event payloads and must@@ -3060,6 +3101,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {errorMessage: message,...classified,...billingFields,+ ...referencedProjectStagingFailuresField,model: prepared.requestedModel || null,clearSession,resultJson: { phase: "ensure_session" },@@ -3079,6 +3121,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {errorMessage: "ACPX did not return a runtime session handle.",errorCode: "acpx_runtime_error",...billingFields,+ ...referencedProjectStagingFailuresField,model: prepared.requestedModel || null,resultJson: { phase: "ensure_session" },summary: "ACPX did not return a runtime session handle.",@@ -3122,6 +3165,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {errorMessage: message,...classified,...billingFields,+ ...referencedProjectStagingFailuresField,model: prepared.requestedModel || null,clearSession,resultJson: {@@ -3322,6 +3366,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {sessionParams: buildSessionParams({ prepared, handle: sessionHandle }),sessionDisplayId: sessionHandle.agentSessionId ?? sessionHandle.backendSessionId ?? sessionHandle.runtimeSessionName,...billingFields,+ ...referencedProjectStagingFailuresField,model: prepared.requestedModel || null,...(turnUsage.usage ? { usage: turnUsage.usage, usageBasis: "per_run" as const } : {}),costUsd: turnUsage.costUsd,@@ -3378,6 +3423,7 @@ export function createAcpxEngineExecutor(deps: AcpxEngineExecutorOptions = {}) {errorCode: timedOut ? "acpx_timeout" : classified.errorCode,errorMeta: classified.errorMeta,...billingFields,+ ...referencedProjectStagingFailuresField,model: prepared.requestedModel || null,clearSession: clearSession || timedOut,resultJson: { phase: "turn" },diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.tsindex 9652fd2cb3..d1d878d7c2 100644--- a/packages/adapter-utils/src/execution-target.ts+++ b/packages/adapter-utils/src/execution-target.ts@@ -14,8 +14,14 @@ import {prepareRemoteManagedRuntime,remoteExecutionSessionMatches,} from "./remote-managed-runtime.js";-import type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js";-export type { SandboxAdditionalSource } from "./sandbox-managed-runtime.js";+import type {+ AdditionalSourceStagingFailure,+ SandboxAdditionalSource,+} from "./sandbox-managed-runtime.js";+export type {+ AdditionalSourceStagingFailure,+ SandboxAdditionalSource,+} from "./sandbox-managed-runtime.js";import {createCommandManagedSandboxCallbackBridgeQueueClient,createSandboxCallbackBridgeAsset,@@ -122,6 +128,12 @@ export interface PreparedAdapterExecutionTargetRuntime {* additional sources were requested.*/additionalSourceDirs: Record<string, string>;+ /**+ * Each additional (referenced) project whose staging failed, paired with the+ * failure message. Empty for a local target, for a transport that does not+ * stage referenced projects, or when every requested project staged.+ */+ additionalSourceFailures: AdditionalSourceStagingFailure[];restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;}@@ -1135,6 +1147,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {runtimeRootDir: null,assetDirs: {},additionalSourceDirs: {},+ additionalSourceFailures: [],restoreWorkspace: async () => {},};}@@ -1157,6 +1170,9 @@ export async function prepareAdapterExecutionTargetRuntime(input: {runtimeRootDir: prepared.runtimeRootDir,assetDirs: prepared.assetDirs,additionalSourceDirs: prepared.additionalSourceDirs,+ // The SSH transport does not stage referenced projects (it is out of scope), so it never+ // reports a per-project staging failure.+ additionalSourceFailures: [],restoreWorkspace: prepared.restoreWorkspace,};}@@ -1192,6 +1208,7 @@ export async function prepareAdapterExecutionTargetRuntime(input: {runtimeRootDir: prepared.runtimeRootDir,assetDirs: prepared.assetDirs,additionalSourceDirs: prepared.additionalSourceDirs,+ additionalSourceFailures: prepared.additionalSourceFailures,restoreWorkspace: prepared.restoreWorkspace,};}diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts b/packages/adapter-utils/src/sandbox-managed-runtime.test.tsindex db796761f2..4e82cb0d86 100644--- a/packages/adapter-utils/src/sandbox-managed-runtime.test.ts+++ b/packages/adapter-utils/src/sandbox-managed-runtime.test.ts@@ -1904,6 +1904,14 @@ describe("sandbox managed runtime", () => {expect(prepared.additionalSourceDirs["proj-second"]).toBe(path.posix.join(runtimeRootDir, "project-proj-second"));expect(prepared.additionalSourceDirs["proj-missing"]).toBeUndefined();+ // The skipped project is a first-class per-project failure outcome, not only a warning, so the+ // run can count it in the requested-vs-synced accounting. The two healthy projects do not+ // appear as failures.+ expect(prepared.additionalSourceFailures.map((failure) => failure.projectId)).toEqual([+ "proj-missing",+ ]);+ expect(prepared.additionalSourceFailures[0]!.error.length).toBeGreaterThan(0);+await expect(readFile(path.join(prepared.additionalSourceDirs["proj-first"], "docs", "guide.md"), "utf8")).resolves.toBe("first guide\n");await expect(readFile(path.join(prepared.additionalSourceDirs["proj-second"], "notes.md"), "utf8"))diff --git a/packages/adapter-utils/src/sandbox-managed-runtime.ts b/packages/adapter-utils/src/sandbox-managed-runtime.tsindex 0cff256464..74b1f4520c 100644--- a/packages/adapter-utils/src/sandbox-managed-runtime.ts+++ b/packages/adapter-utils/src/sandbox-managed-runtime.ts@@ -300,9 +300,23 @@ export interface PreparedSandboxManagedRuntime {* were requested.*/additionalSourceDirs: Record<string, string>;+ /**+ * Each additional (referenced) project whose staging failed, paired with the+ * failure message. Per-project failure isolation keeps one project's failure+ * from aborting the run, so a failed project is absent from+ * `additionalSourceDirs` and present here. Empty when every requested project+ * staged, or when no additional sources were requested.+ */+ additionalSourceFailures: AdditionalSourceStagingFailure[];restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;}+/** One additional (referenced) project that failed to stage into the sandbox. */+export interface AdditionalSourceStagingFailure {+ projectId: string;+ error: string;+}+function asObject(value: unknown): Record<string, unknown> {return value && typeof value === "object" && !Array.isArray(value)? (value as Record<string, unknown>)@@ -736,6 +750,10 @@ export async function prepareSandboxManagedRuntime(input: {// Remote directory of each additional (referenced) project that stages// successfully, keyed by projectId. A project that fails to stage is absent.const additionalSourceDirs: Record<string, string> = {};+ // Each additional (referenced) project whose staging failed, paired with the+ // failure message. Per-project failure isolation keeps the run and the other+ // projects going; this list makes each failure a first-class, reported outcome.+ const additionalSourceFailures: AdditionalSourceStagingFailure[] = [];// Additional projects stage as plain trees. Drop the heavy build/cache dirs a// reference tree does not need, and `.git` — additional sources never carry// git-history semantics (anchor-only).@@ -999,9 +1017,13 @@ export async function prepareSandboxManagedRuntime(input: {});additionalSourceDirs[projectId] = remoteProjectDir;} catch (error) {+ const message = error instanceof Error ? error.message : String(error);console.warn(- `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${String(error)}`,+ `[paperclip] Failed to stage referenced project ${projectId}; skipping it. ${message}`,);+ // Record the failure as a first-class per-project outcome so the run can count it in the+ // requested-vs-synced accounting instead of losing it to a warning line.+ additionalSourceFailures.push({ projectId, error: message });}}});@@ -1017,6 +1039,7 @@ export async function prepareSandboxManagedRuntime(input: {runtimeRootDir,assetDirs,additionalSourceDirs,+ additionalSourceFailures,restoreWorkspace: async (onProgress?: RuntimeProgressSink) => {const restoreSink = onProgress ?? input.onProgress;if (!syncWorkspace) {diff --git a/packages/adapter-utils/src/server-utils.test.ts b/packages/adapter-utils/src/server-utils.test.tsindex 4d4c9d6523..74f032a968 100644--- a/packages/adapter-utils/src/server-utils.test.ts+++ b/packages/adapter-utils/src/server-utils.test.ts@@ -2295,6 +2295,53 @@ describe("shapePaperclipWorkspaceEnvForExecution", () => {});});+ it("repoints a referenced hint to its staged remote directory when the map has an entry", () => {+ const shaped = shapePaperclipWorkspaceEnvForExecution({+ workspaceCwd: "/tmp/workspace",+ workspaceWorktreePath: "/tmp/worktree",+ workspaceHints: [+ // The anchor hint keeps its remote-cwd rewrite.+ { workspaceId: "workspace-1", cwd: "/tmp/workspace" },+ // A referenced hint with a staged directory repoints at it.+ { workspaceId: "workspace-2", cwd: "/tmp/referenced/project-a", projectId: "project-a" },+ // A referenced hint with no staged directory loses its cwd.+ { workspaceId: "workspace-3", cwd: "/tmp/referenced/project-b", projectId: "project-b" },+ ],+ executionTargetIsRemote: true,+ executionCwd: "/remote/workspace",+ stagedProjectDirs: { "project-a": "/remote/runtime/project-project-a" },+ });++ expect(shaped).toEqual({+ workspaceCwd: "/remote/workspace",+ workspaceWorktreePath: null,+ workspaceHints: [+ { workspaceId: "workspace-1", cwd: "/remote/workspace" },+ {+ workspaceId: "workspace-2",+ cwd: "/remote/runtime/project-project-a",+ projectId: "project-a",+ },+ { workspaceId: "workspace-3", projectId: "project-b" },+ ],+ });+ });++ it("removes cwd from a referenced hint that has no staged directory", () => {+ const shaped = shapePaperclipWorkspaceEnvForExecution({+ workspaceCwd: "/tmp/workspace",+ workspaceHints: [+ { workspaceId: "workspace-2", cwd: "/tmp/referenced/project-a", projectId: "project-a" },+ ],+ executionTargetIsRemote: true,+ executionCwd: "/remote/workspace",+ // The map is empty, so the referenced hint has no staged directory.+ stagedProjectDirs: {},+ });++ expect(shaped.workspaceHints).toEqual([{ workspaceId: "workspace-2", projectId: "project-a" }]);+ });+it("leaves local execution workspace paths unchanged", () => {const workspaceHints = [{ workspaceId: "workspace-1", cwd: "/tmp/workspace" }];const shaped = shapePaperclipWorkspaceEnvForExecution({diff --git a/packages/adapter-utils/src/server-utils.ts b/packages/adapter-utils/src/server-utils.tsindex 5a3fc36cbb..41d96a2349 100644--- a/packages/adapter-utils/src/server-utils.ts+++ b/packages/adapter-utils/src/server-utils.ts@@ -2026,6 +2026,15 @@ export function shapePaperclipWorkspaceEnvForExecution(input: {workspaceHints?: Array<Record<string, unknown>>;executionTargetIsRemote?: boolean;executionCwd?: string | null;+ /**+ * On a remote target, the map of referenced (mentioned) project id to the staged in-sandbox+ * directory that received that project's tree (`project-<projectId>`). A non-anchor hint whose+ * `projectId` has an entry repoints its `cwd` to the staged directory. A non-anchor hint with no+ * entry loses its `cwd`, so the agent never receives a path the transport did not stage. The map+ * is empty on a local target and defaults to empty, so a caller that passes nothing keeps the+ * previous behavior (every non-anchor hint loses its `cwd` on a remote target).+ */+ stagedProjectDirs?: Record<string, string>;}): {workspaceCwd: string | null;workspaceWorktreePath: string | null;@@ -2068,6 +2077,7 @@ export function shapePaperclipWorkspaceEnvForExecution(input: {}const realizedWorkspaceCwd = executionCwd;const localWorkspaceCwd = workspaceCwd ? path.resolve(workspaceCwd) : null;+ const stagedProjectDirs = input.stagedProjectDirs ?? {};const shapedWorkspaceHints = workspaceHints.map((hint) => {const nextHint = { ...hint };const hintCwd = typeof nextHint.cwd === "string" ? nextHint.cwd.trim() : "";@@ -2082,7 +2092,19 @@ export function shapePaperclipWorkspaceEnvForExecution(input: {return nextHint;}- delete nextHint.cwd;+ // A referenced (mentioned) project hint carries its `projectId`. When the transport staged that+ // project into the sandbox, repoint the hint at its staged `project-<projectId>` directory so+ // the agent reads it there. Without a staged directory the hint would point at a local path that+ // the remote target cannot reach, so remove the `cwd` (fail loud — never expose an unstaged+ // path). This also removes the `cwd` of a non-anchor hint that carries no `projectId`, such as an+ // alternative anchor-project workspace, which keeps the previous behavior for those hints.+ const hintProjectId = typeof nextHint.projectId === "string" ? nextHint.projectId : "";+ const stagedProjectDir = hintProjectId ? stagedProjectDirs[hintProjectId] : undefined;+ if (stagedProjectDir && stagedProjectDir.trim().length > 0) {+ nextHint.cwd = stagedProjectDir.trim();+ } else {+ delete nextHint.cwd;+ }return nextHint;});@@ -2145,6 +2167,8 @@ export function refreshPaperclipWorkspaceEnvForExecution(input: {agentHome?: string | null;executionTargetIsRemote?: boolean;executionCwd?: string | null;+ /** Referenced-project id to staged in-sandbox directory map; see {@link shapePaperclipWorkspaceEnvForExecution}. */+ stagedProjectDirs?: Record<string, string>;}): {workspaceCwd: string | null;workspaceWorktreePath: string | null;@@ -2156,6 +2180,7 @@ export function refreshPaperclipWorkspaceEnvForExecution(input: {workspaceHints: input.workspaceHints,executionTargetIsRemote: input.executionTargetIsRemote,executionCwd: input.executionCwd,+ stagedProjectDirs: input.stagedProjectDirs,});delete input.env.PAPERCLIP_WORKSPACE_CWD;diff --git a/packages/adapter-utils/src/types.ts b/packages/adapter-utils/src/types.tsindex 988e1318e7..d065722def 100644--- a/packages/adapter-utils/src/types.ts+++ b/packages/adapter-utils/src/types.ts@@ -103,6 +103,14 @@ export interface AdapterExecutionResult {costUsd?: number | null;resultJson?: Record<string, unknown> | null;runtimeServices?: AdapterRuntimeServiceReport[];+ /**+ * Each referenced (mentioned) project that failed to stage into the remote sandbox for this run,+ * by `projectId`. The run continues without a failed project (per-project failure isolation); this+ * field carries the failure back so the server counts it in the requested-vs-synced observability+ * instead of losing it to a warning line. Absent or empty on a local target, or when every staged+ * referenced project succeeded.+ */+ referencedProjectStagingFailures?: Array<{ projectId: string }>;summary?: string | null;clearSession?: boolean;question?: {diff --git a/packages/plugins/sdk/src/protocol.ts b/packages/plugins/sdk/src/protocol.tsindex f85446a83c..91d1d58f66 100644--- a/packages/plugins/sdk/src/protocol.ts+++ b/packages/plugins/sdk/src/protocol.ts@@ -638,6 +638,12 @@ export interface PluginEnvironmentRealizeWorkspaceParams extends PluginEnvironme};}+/**+ * A plugin `environmentRealizeWorkspace` handler returns only the realized cwd and provider+ * metadata. The server, not the plugin, builds the full workspace-realization record from the run+ * request and merges this cwd and metadata into it. Do not return a `workspaceRealization` record+ * here; the server owns that record, so the referenced (mentioned) project sources reach the adapter.+ */export interface PluginEnvironmentRealizeWorkspaceResult {cwd: string;metadata?: Record<string, unknown>;diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.tsindex cefe00fc8b..3c836883ab 100644--- a/server/src/__tests__/environment-runtime.test.ts+++ b/server/src/__tests__/environment-runtime.test.ts@@ -699,6 +699,231 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentReleaseLease", expect.anything(), 31234);});+ it("builds the workspace-realization record with referenced sources for a plugin-backed sandbox realize", async () => {+ // A provider plugin realize handler returns only its realized cwd and provider metadata; it does+ // not build the workspace-realization record. The server must build that record from the run+ // request, so the referenced (mentioned) project sources reach the adapter through+ // `realization.additional`. Without the record the sandbox agent never receives the mentioned+ // projects. This test drives the plugin-backed sandbox realize path and asserts the referenced+ // source survives into the returned record.+ const pluginId = randomUUID();+ const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();+ const fakePluginConfig = {+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: false,+ };+ const environment = {+ ...baseEnvironment,+ name: "Fake Plugin Sandbox Realize",+ driver: "sandbox",+ config: fakePluginConfig,+ };+ await environmentService(db).update(environment.id, {+ driver: "sandbox",+ name: environment.name,+ config: fakePluginConfig,+ });+ await db.insert(plugins).values({+ id: pluginId,+ pluginKey: "paperclip.fake-plugin-sandbox-provider",+ packageName: "@paperclipai/plugin-fake-sandbox",+ version: "1.0.0",+ apiVersion: 1,+ categories: ["automation"],+ manifestJson: {+ id: "paperclip.fake-plugin-sandbox-provider",+ apiVersion: 1,+ version: "1.0.0",+ displayName: "Fake Plugin Sandbox Provider",+ description: "Test fake plugin provider",+ author: "Paperclip",+ categories: ["automation"],+ capabilities: ["environment.drivers.register"],+ entrypoints: { worker: "dist/worker.js" },+ environmentDrivers: [+ {+ driverKey: "fake-plugin",+ kind: "sandbox_provider",+ displayName: "Fake Plugin",+ configSchema: { type: "object" },+ },+ ],+ },+ status: "ready",+ installOrder: 1,+ updatedAt: new Date(),+ } as any);+ const workerManager = {+ isRunning: vi.fn((id: string) => id === pluginId),+ call: vi.fn(async (_pluginId: string, method: string) => {+ if (method === "environmentAcquireLease") {+ return {+ providerLeaseId: "sandbox-realize-1",+ metadata: {+ provider: "fake-plugin",+ image: "fake:test",+ timeoutMs: 1234,+ reuseLease: false,+ remoteCwd: "/workspace",+ },+ };+ }+ if (method === "environmentRealizeWorkspace") {+ // Mimic a real provider (for example Daytona): return only the realized cwd and provider+ // metadata, never a `workspaceRealization` record.+ return {+ cwd: "/workspace/project",+ metadata: {+ provider: "fake-plugin",+ remoteCwd: "/workspace/project",+ },+ };+ }+ throw new Error(`Unexpected plugin method: ${method}`);+ }),+ } as unknown as PluginWorkerManager;+ const runtimeWithPlugin = environmentRuntimeService(db, { pluginWorkerManager: workerManager });++ const acquired = await runtimeWithPlugin.acquireRunLease({+ companyId,+ environment,+ issueId: null,+ heartbeatRunId: runId,+ persistedExecutionWorkspace: null,+ });+ const workspaceRealizationRequest = {+ version: 1,+ adapterType: "codex_local",+ companyId,+ environmentId: environment.id,+ executionWorkspaceId: null,+ issueId: null,+ heartbeatRunId: runId,+ requestedMode: "ephemeral",+ source: {+ kind: "project_primary",+ localPath: "/tmp/anchor",+ projectId: "anchor-project",+ projectWorkspaceId: "anchor-workspace",+ repoUrl: null,+ repoRef: null,+ strategy: "project_primary",+ branchName: null,+ worktreePath: null,+ },+ additionalSources: [+ {+ localPath: "/tmp/referenced-project",+ projectId: "referenced-project-1",+ projectWorkspaceId: "referenced-workspace-1",+ repoUrl: null,+ repoRef: null,+ },+ ],+ };+ const realized = await runtimeWithPlugin.realizeWorkspace({+ environment,+ lease: acquired.lease,+ workspace: {+ localPath: "/tmp/anchor",+ mode: "ephemeral",+ metadata: { workspaceRealizationRequest },+ },+ });++ // The provider realized cwd and provider metadata survive.+ expect(realized.cwd).toBe("/workspace/project");+ expect(realized.metadata?.provider).toBe("fake-plugin");+ // The server-built record carries the referenced source through `additional`, so the adapter can+ // stage the mentioned project into the sandbox.+ const realization = realized.metadata?.workspaceRealization as Record<string, unknown> | undefined;+ expect(realization).toBeDefined();+ expect(realization?.additional).toEqual([+ expect.objectContaining({+ path: "/tmp/referenced-project",+ projectId: "referenced-project-1",+ }),+ ]);+ });++ it("builds the workspace-realization record with referenced sources for a built-in sandbox realize", async () => {+ // The sandbox driver `realizeWorkspace` has two exits: a plugin-backed provider and a+ // built-in provider. Both must build the same workspace-realization record, so the+ // referenced (mentioned) project sources reach the adapter through `realization.additional`.+ // The test above covers the plugin exit. This test covers the built-in exit (no provider+ // plugin), so a regression on either exit fails a test.+ const { companyId, environment, runId } = await seedEnvironment({+ driver: "sandbox",+ name: "Fake Sandbox Realize",+ config: {+ provider: "fake",+ image: "ubuntu:24.04",+ reuseLease: true,+ },+ });++ const acquired = await runtime.acquireRunLease({+ companyId,+ environment,+ issueId: null,+ heartbeatRunId: runId,+ persistedExecutionWorkspace: null,+ });++ const workspaceRealizationRequest = {+ version: 1,+ adapterType: "codex_local",+ companyId,+ environmentId: environment.id,+ executionWorkspaceId: null,+ issueId: null,+ heartbeatRunId: runId,+ requestedMode: "ephemeral",+ source: {+ kind: "project_primary",+ localPath: "/tmp/anchor",+ projectId: "anchor-project",+ projectWorkspaceId: "anchor-workspace",+ repoUrl: null,+ repoRef: null,+ strategy: "project_primary",+ branchName: null,+ worktreePath: null,+ },+ additionalSources: [+ {+ localPath: "/tmp/referenced-project",+ projectId: "referenced-project-1",+ projectWorkspaceId: "referenced-workspace-1",+ repoUrl: null,+ repoRef: null,+ },+ ],+ };+ const realized = await runtime.realizeWorkspace({+ environment,+ lease: acquired.lease,+ workspace: {+ localPath: "/tmp/anchor",+ mode: "ephemeral",+ metadata: { workspaceRealizationRequest },+ },+ });++ // The built-in exit builds the record and carries the referenced source through `additional`,+ // so the adapter can stage the mentioned project into the sandbox.+ const realization = realized.metadata?.workspaceRealization as Record<string, unknown> | undefined;+ expect(realization).toBeDefined();+ expect(realization?.additional).toEqual([+ expect.objectContaining({+ path: "/tmp/referenced-project",+ projectId: "referenced-project-1",+ }),+ ]);+ });+it("uses resolved secret-ref config for plugin-backed sandbox execute and release", async () => {const pluginId = randomUUID();const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();diff --git a/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts b/server/src/__tests__/heartbeat-remote-referenced-projects.test.tsnew file mode 100644index 0000000000..de2403f820--- /dev/null+++ b/server/src/__tests__/heartbeat-remote-referenced-projects.test.ts@@ -0,0 +1,359 @@+import { randomUUID } from "node:crypto";+import { describe, expect, it } from "vitest";+import {+ isConfinedRemoteStagingDriver,+ isMultiProjectWorkspaceSyncRemoteEnabled,+ MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS,+ MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS,+ MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV,+ resolveAdditionalRunWorkspaces,+ type ResolveAdditionalRunWorkspacesOptions,+ type ResolvedAdditionalWorkspace,+} from "../services/heartbeat.ts";+import type { AuthorizationActor, AuthorizationDecision } from "../services/authorization.ts";++// These tests exercise the remote gate in `resolveAdditionalRunWorkspaces` with fully injected+// dependencies (no database). The function reads mentions through `issues.findMentionedProjectIds`,+// hydrates candidates through `projects.listByIds`, authorizes each through `access.decide`, and+// resolves an admitted project to a workspace through `resolveProjectWorkspace`. Each dependency is+// a stub here, so a test asserts the exact gate behavior for a sandbox target, an SSH target, and+// the remote kill switch.++const buildActor = (companyId: string): AuthorizationActor => ({+ type: "agent",+ agentId: randomUUID(),+ companyId,+ source: "agent_key",+});++const decision = (allowed: boolean): AuthorizationDecision => ({+ allowed,+ action: "project:read",+ reason: allowed ? "allow_company_agent" : "deny_company_boundary",+ explanation: "test decision",+});++// A `projects.listByIds` stub that returns one minimal record per requested id, in request order.+// The referenced-project record only needs an `id` here; the workspace resolution is stubbed, so no+// other field is read. The cast keeps the stub minimal without restating the full project row shape.+const listByIdsStub: ResolveAdditionalRunWorkspacesOptions["projects"] = {+ listByIds: async (_companyId, ids) =>+ ids.map(+ (id) =>+ ({ id, name: `Project ${id}`, status: "in_progress" }) as Awaited<+ ReturnType<ResolveAdditionalRunWorkspacesOptions["projects"]["listByIds"]>+ >[number],+ ),+};++// A `findMentionedProjectIds` stub that returns a fixed mention set for any issue.+const mentionsStub = (+ mentionedProjectIds: string[],+): ResolveAdditionalRunWorkspacesOptions["issues"] => ({+ findMentionedProjectIds: async () => mentionedProjectIds,+});++// Records every `project:read` authorization call and answers via the supplied resolver.+const recordingAccess = (+ resolve: (projectId: string) => AuthorizationDecision,+): { decidedProjectIds: string[]; access: ResolveAdditionalRunWorkspacesOptions["access"] } => {+ const decidedProjectIds: string[] = [];+ const access: ResolveAdditionalRunWorkspacesOptions["access"] = {+ decide: async (input) => {+ const resource = input.resource;+ const projectId = resource.type === "project" ? (resource.projectId ?? "") : "";+ decidedProjectIds.push(projectId);+ return resolve(projectId);+ },+ };+ return { decidedProjectIds, access };+};++// Records every workspace that resolution stages and returns a read-only workspace stub for it.+const recordingResolveProjectWorkspace = (): {+ stagedProjectIds: string[];+ resolveProjectWorkspace: ResolveAdditionalRunWorkspacesOptions["resolveProjectWorkspace"];+} => {+ const stagedProjectIds: string[] = [];+ const resolveProjectWorkspace: ResolveAdditionalRunWorkspacesOptions["resolveProjectWorkspace"] =+ async (project) => {+ stagedProjectIds.push(project.projectId);+ return {+ cwd: `/tmp/referenced/${project.projectId}`,+ projectId: project.projectId,+ workspaceId: null,+ repoUrl: null,+ repoRef: null,+ } satisfies ResolvedAdditionalWorkspace;+ };+ return { stagedProjectIds, resolveProjectWorkspace };+};++const baseOpts = (+ companyId: string,+ overrides: Partial<ResolveAdditionalRunWorkspacesOptions> &+ Pick<ResolveAdditionalRunWorkspacesOptions, "issues" | "access" | "resolveProjectWorkspace">,+): ResolveAdditionalRunWorkspacesOptions => ({+ enabled: true,+ companyId,+ actor: buildActor(companyId),+ projects: listByIdsStub,+ ...overrides,+});++describe("remote referenced-project kill switch", () => {+ it("is ON by default and disabled only by an explicit false env value", () => {+ // Default ON: an unset value resolves the remote path live (go-live default).+ expect(isMultiProjectWorkspaceSyncRemoteEnabled({})).toBe(true);+ // The targeted kill switch: an explicit false value (the rollback path) disables it.+ expect(+ isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "" }),+ ).toBe(false);+ expect(+ isMultiProjectWorkspaceSyncRemoteEnabled({+ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "false",+ }),+ ).toBe(false);+ expect(+ isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "0" }),+ ).toBe(false);+ expect(+ isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "off" }),+ ).toBe(false);+ // Any other value keeps the remote path on.+ expect(+ isMultiProjectWorkspaceSyncRemoteEnabled({ [MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]: "true" }),+ ).toBe(true);+ });++ it("classifies only the sandbox driver as a confined remote staging transport", () => {+ // Only the sandbox driver confines each staged referenced tree, so only it opens the gate.+ expect(isConfinedRemoteStagingDriver("sandbox")).toBe(true);+ // The SSH and plugin drivers keep dropping referenced projects.+ expect(isConfinedRemoteStagingDriver("ssh")).toBe(false);+ expect(isConfinedRemoteStagingDriver("plugin")).toBe(false);+ expect(isConfinedRemoteStagingDriver("local")).toBe(false);+ expect(isConfinedRemoteStagingDriver(null)).toBe(false);+ expect(isConfinedRemoteStagingDriver(undefined)).toBe(false);+ });+});++describe("resolveAdditionalRunWorkspaces remote gate", () => {+ it("resolves and authorizes referenced workspaces on a sandbox target with the remote flag on", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ const mentionedProjectId = randomUUID();+ const { decidedProjectIds, access } = recordingAccess(() => decision(true));+ const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ const result = await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ targetStagesConfined: true,+ remoteReferencedSyncEnabled: true,+ issues: mentionsStub([mentionedProjectId]),+ access,+ resolveProjectWorkspace,+ }),+ );++ // The run authorized the referenced project against the run actor and staged its workspace.+ expect(decidedProjectIds).toEqual([mentionedProjectId]);+ expect(stagedProjectIds).toEqual([mentionedProjectId]);+ expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([+ mentionedProjectId,+ ]);+ expect(result.failures).toEqual([]);+ });++ it("fails closed on a sandbox target when the remote flag is off (no authorization, no staging)", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ const mentionedProjectId = randomUUID();+ const { decidedProjectIds, access } = recordingAccess(() => decision(true));+ const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ const result = await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ targetStagesConfined: true,+ remoteReferencedSyncEnabled: false,+ issues: mentionsStub([mentionedProjectId]),+ access,+ resolveProjectWorkspace,+ }),+ );++ // Fail closed: no authorization decision and no workspace staging ran.+ expect(decidedProjectIds).toEqual([]);+ expect(stagedProjectIds).toEqual([]);+ expect(result.additionalWorkspaces).toEqual([]);+ // The whole referenced set is still counted as a staging-layer drop.+ expect(result.failures).toEqual([{ projectId: mentionedProjectId, reason: "staging" }]);+ });++ it("drops referenced projects on an SSH target whether the remote flag is on or off", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ const mentionedProjectId = randomUUID();++ for (const remoteReferencedSyncEnabled of [true, false]) {+ const { decidedProjectIds, access } = recordingAccess(() => decision(true));+ const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ const result = await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ // The SSH transport does not confine its staging path, so it stays out of scope.+ targetStagesConfined: false,+ remoteReferencedSyncEnabled,+ issues: mentionsStub([mentionedProjectId]),+ access,+ resolveProjectWorkspace,+ }),+ );++ expect(decidedProjectIds).toEqual([]);+ expect(stagedProjectIds).toEqual([]);+ expect(result.additionalWorkspaces).toEqual([]);+ expect(result.failures).toEqual([{ projectId: mentionedProjectId, reason: "staging" }]);+ }+ });++ it("drops a referenced project the run agent cannot read on a sandbox target", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ const readableProjectId = randomUUID();+ const unreadableProjectId = randomUUID();+ // Deny project:read for the unreadable project; allow it for the readable one.+ const { decidedProjectIds, access } = recordingAccess((projectId) =>+ decision(projectId === readableProjectId),+ );+ const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ const result = await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ targetStagesConfined: true,+ remoteReferencedSyncEnabled: true,+ issues: mentionsStub([readableProjectId, unreadableProjectId]),+ access,+ resolveProjectWorkspace,+ }),+ );++ // Both projects were authorized against the run actor, but only the readable one was staged.+ expect(decidedProjectIds).toEqual([readableProjectId, unreadableProjectId]);+ expect(stagedProjectIds).toEqual([readableProjectId]);+ expect(result.additionalWorkspaces.map((workspace) => workspace.projectId)).toEqual([+ readableProjectId,+ ]);+ // The denied project is a first-class authorization failure.+ expect(result.failures).toEqual([{ projectId: unreadableProjectId, reason: "authorization" }]);+ });++ it("never authorizes the anchor project on a sandbox target", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ const mentionedProjectId = randomUUID();+ const { decidedProjectIds, access } = recordingAccess(() => decision(true));+ const { resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ targetStagesConfined: true,+ remoteReferencedSyncEnabled: true,+ // The mention set includes the anchor itself; the anchor must never be re-authorized.+ issues: mentionsStub([anchorProjectId, mentionedProjectId]),+ access,+ resolveProjectWorkspace,+ }),+ );++ // The anchor exemption drops the anchor from the mention set before the per-project check, so+ // only the non-anchor referenced project is authorized.+ expect(decidedProjectIds).toEqual([mentionedProjectId]);+ expect(decidedProjectIds).not.toContain(anchorProjectId);+ });++ it("holds the admitted-project cap on a sandbox target", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ // One more mention than the admitted cap.+ const mentionedProjectIds = Array.from(+ { length: MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS + 1 },+ () => randomUUID(),+ );+ const { access } = recordingAccess(() => decision(true));+ const { stagedProjectIds, resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ const result = await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ targetStagesConfined: true,+ remoteReferencedSyncEnabled: true,+ issues: mentionsStub(mentionedProjectIds),+ access,+ resolveProjectWorkspace,+ }),+ );++ // The cap bounds how many additional projects a sandbox run materializes.+ expect(stagedProjectIds).toHaveLength(MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS);+ expect(result.additionalWorkspaces).toHaveLength(MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS);+ // The one project past the cap is a first-class resolution failure.+ expect(result.failures).toEqual([+ { projectId: mentionedProjectIds[MAX_RUN_REFERENCED_ADDITIONAL_PROJECTS], reason: "resolution" },+ ]);+ });++ it("holds the candidate-evaluation cap on a sandbox target", async () => {+ const companyId = randomUUID();+ const issueId = randomUUID();+ const anchorProjectId = randomUUID();+ // Flood the run with denied mentions past the evaluation cap. Every candidate is denied, so the+ // admitted cap is never reached; the evaluation cap must still bound the authorization fan-out.+ const mentionedProjectIds = Array.from(+ { length: MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS + 5 },+ () => randomUUID(),+ );+ const { decidedProjectIds, access } = recordingAccess(() => decision(false));+ const { resolveProjectWorkspace } = recordingResolveProjectWorkspace();++ const result = await resolveAdditionalRunWorkspaces(+ issueId,+ anchorProjectId,+ baseOpts(companyId, {+ executionTargetIsRemote: true,+ targetStagesConfined: true,+ remoteReferencedSyncEnabled: true,+ issues: mentionsStub(mentionedProjectIds),+ access,+ resolveProjectWorkspace,+ }),+ );++ // The evaluation cap bounds authorization decisions even under a denied-mention flood.+ expect(decidedProjectIds).toHaveLength(MAX_RUN_REFERENCED_CANDIDATE_EVALUATIONS);+ expect(result.additionalWorkspaces).toEqual([]);+ });+});diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.tsindex 7e7ab0cd67..7457718a50 100644--- a/server/src/services/environment-runtime.ts+++ b/server/src/services/environment-runtime.ts@@ -1202,7 +1202,14 @@ function createSandboxEnvironmentDriver(},async realizeWorkspace(input) {- // Plugin-backed sandbox providers: delegate workspace realization.+ // Resolve the realized cwd and any provider metadata first, then build ONE+ // workspace-realization record and wrap it the SAME way for every driver. A+ // plugin-backed sandbox provider realizes the workspace remotely and returns its+ // own cwd and metadata. A built-in driver has no plugin call; it uses the lease+ // `remoteCwd`. Both paths must produce the record through the single build below,+ // so the record can never drift between two exits.+ let pluginRealizedCwd: string | null = null;+ let providerMetadata: Record<string, unknown> | null = null;if (input.lease.metadata?.sandboxProviderPlugin && pluginWorkerManager) {const pluginId = readString(input.lease.metadata?.pluginId);const providerKey =@@ -1216,7 +1223,7 @@ function createSandboxEnvironmentDriver(lease: input.lease,provider: providerKey,});- return await pluginWorkerManager.call(pluginId, "environmentRealizeWorkspace", {+ const pluginResult = await pluginWorkerManager.call(pluginId, "environmentRealizeWorkspace", {driverKey: providerKey,companyId: input.lease.companyId,environmentId: input.environment.id,@@ -1229,21 +1236,36 @@ function createSandboxEnvironmentDriver(},workspace: input.workspace,}, resolvePluginSandboxRpcTimeoutMs(stripSandboxProviderEnvelope(config as SandboxEnvironmentConfig)));+ pluginRealizedCwd =+ typeof pluginResult.cwd === "string" && pluginResult.cwd.trim().length > 0+ ? pluginResult.cwd.trim()+ : null;+ providerMetadata = pluginResult.metadata ?? null;}}+ // A plugin realize handler returns only its realized cwd and provider metadata; it+ // does not build the full workspace-realization record. The server builds that record+ // from the run request, so the referenced (mentioned) project sources reach the adapter+ // through `realization.additional`. The adapter reads that field to stage each referenced+ // tree into the sandbox; without the record the sandbox agent never receives the mentioned+ // projects. The provider cwd and metadata still drive the remote path when a plugin realizes+ // the workspace.const record = buildWorkspaceRealizationRecordFromDriverInput({environment: input.environment,lease: input.lease,workspace: input.workspace,cwd:- typeof input.lease.metadata?.remoteCwd === "string" && input.lease.metadata.remoteCwd.trim().length > 0+ pluginRealizedCwd ??+ (typeof input.lease.metadata?.remoteCwd === "string" && input.lease.metadata.remoteCwd.trim().length > 0? input.lease.metadata.remoteCwd.trim()- : input.workspace.remotePath ?? input.workspace.localPath ?? null,+ : input.workspace.remotePath ?? input.workspace.localPath ?? null),+ providerMetadata,});return {- cwd: record.remote.path ?? record.local.path,+ cwd: pluginRealizedCwd ?? record.remote.path ?? record.local.path,metadata: {+ ...(providerMetadata ?? {}),workspaceRealization: record,},};diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex bcc54e70c3..08d86a7672 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -2377,6 +2377,41 @@ export function isRemoteExecutionEnvironmentDriver(driver: string | null | undefreturn driver === "ssh" || driver === "sandbox" || driver === "plugin";}+/**+ * Environment flag (kill-switch, default ON) that gates whether a *remote* run stages the+ * referenced (mentioned) project set into the sandbox. This is a targeted rollback lever: it+ * disables only the remote referenced-project path and never regresses the working local path.+ * The master flag {@link MULTI_PROJECT_WORKSPACE_SYNC_ENV} is the blunt switch that kills both+ * local and remote. The remote path runs when both the master flag and this remote flag are ON —+ * the default state. An unset value resolves ON; an operator disables it with an explicit false+ * value (`"false"`, `"0"`, `"off"`, `"no"`, or `""`). The OFF state fails closed: a remote run+ * runs no referenced-project authorization or staging and reverts to the remote drop path.+ */+export const MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV =+ "PAPERCLIP_MULTI_PROJECT_WORKSPACE_SYNC_REMOTE";++export function isMultiProjectWorkspaceSyncRemoteEnabled(+ env: Record<string, string | undefined> = process.env,+): boolean {+ // Default ON: an unset value is not false, so the remote path is live unless an operator sets+ // an explicit false value as the targeted kill switch (rollback path).+ return !isFalsyRuntimeEnvValue(env[MULTI_PROJECT_WORKSPACE_SYNC_REMOTE_ENV]);+}++/**+ * True when an environment driver stages a multi-source remote workspace through the confined+ * sandbox/command runtime. Only the `sandbox` driver asserts per-project confinement on the+ * staging path (`assertSyncOperationsConfined` in `sandbox-managed-runtime`). The `ssh` driver+ * stages without that guard, and the `plugin` driver does not route through the confined command+ * runtime in the workspace-realization step, so both keep dropping referenced projects. A `local`+ * (or unknown) driver is not remote and never reaches this check. This gate is intentionally+ * narrower than {@link isRemoteExecutionEnvironmentDriver}: it names the one transport that+ * confines each staged referenced tree.+ */+export function isConfinedRemoteStagingDriver(driver: string | null | undefined): boolean {+ return driver === "sandbox";+}+/*** Upper bound on how many additional (mentioned) projects a single run may materialize* beyond the anchor. Bounds the fan-out of per-project authorization and workspace prep.@@ -2647,11 +2682,26 @@ export interface ResolveAdditionalRunWorkspacesOptions {maxCandidateEvaluations?: number;/*** True when the run executes on a non-local target (ssh, sandbox, or plugin). A referenced- * project realizes as a local directory only, and a remote target has no path yet to receive- * that tree, so a resolved cwd would not exist on the target. When true, the function skips- * referenced-project authorization and workspace work and returns no additional workspaces.+ * project realizes as a local directory first. On a remote target that local tree reaches the+ * agent only when a confined transport stages it into the sandbox and the remote flag is on+ * (see `targetStagesConfined` and `remoteReferencedSyncEnabled`). Otherwise the run drops the+ * whole referenced set and records it at the staging layer.*/executionTargetIsRemote?: boolean;+ /**+ * True when the remote target stages each referenced tree through the confined sandbox/command+ * runtime (the `sandbox` driver; see {@link isConfinedRemoteStagingDriver}). The gate opens the+ * referenced-project path on a remote target only when this is true. The SSH transport and any+ * unconfined transport keep dropping referenced projects. Ignored on a local target.+ */+ targetStagesConfined?: boolean;+ /**+ * The remote-only kill switch (default ON; see {@link isMultiProjectWorkspaceSyncRemoteEnabled}).+ * When true, a confined remote target stages the referenced set. When false, a remote target+ * fails closed: it runs no referenced-project authorization or staging and reverts to the remote+ * drop path. Ignored on a local target.+ */+ remoteReferencedSyncEnabled?: boolean;}/**@@ -2677,34 +2727,44 @@ export async function resolveAdditionalRunWorkspaces(return { additionalWorkspaces: [], warnings: [], failures: [] };}- // A referenced project realizes as a local directory only. A remote execution target (ssh,- // sandbox, or plugin) has no path yet to receive the referenced tree, so a resolved cwd would- // not exist on the target and the anchor-only remote sync never carries it across. Skip the- // referenced-project authorization and clone work on a remote target, so the run neither does- // work it must discard nor exposes an inaccessible referenced path to the agent. Warn only when- // the issue actually mentions a project, so a remote run without any referenced mention stays- // silent.+ // A referenced project realizes as a local directory first. On a remote target the run carries+ // that tree to the agent only when a confined transport stages it into the sandbox and the+ // remote flag is on. The confined sandbox transport asserts per-project confinement on each+ // staged tree (`assertSyncOperationsConfined` in `sandbox-managed-runtime`). The SSH transport+ // does not, so it stays out of scope. When the remote flag is off the run fails closed. In every+ // one of those drop cases the run neither does authorization or clone work it must discard nor+ // exposes an inaccessible referenced path to the agent.if (opts.executionTargetIsRemote) {- const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, {- includeCommentBodies: true,- });- // Each distinct non-anchor mention is a referenced project this remote run drops. A remote- // target has no path to receive the referenced tree, so the run drops the whole set at the- // staging layer. Record one failure per dropped project so the requested-vs-synced accounting- // counts the whole referenced set and the run still emits its structured sync log.- const droppedProjectIds = [- ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)),- ];- return {- additionalWorkspaces: [],- warnings:- droppedProjectIds.length > 0- ? [- "Referenced-project workspaces are available only on a local execution target. This run uses a remote execution target, so no referenced-project workspace was attached.",- ]- : [],- failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })),- };+ const remoteReferencedSyncOpen =+ (opts.remoteReferencedSyncEnabled ?? false) && (opts.targetStagesConfined ?? false);+ if (!remoteReferencedSyncOpen) {+ const mentionedIds = await opts.issues.findMentionedProjectIds(issueId, {+ includeCommentBodies: true,+ });+ // Each distinct non-anchor mention is a referenced project this remote run drops. An SSH+ // target (or the remote flag off) has no confined path to receive the referenced tree, so+ // the run drops the whole set at the staging layer. Record one failure per dropped project+ // so the requested-vs-synced accounting counts the whole referenced set and the run still+ // emits its structured sync log. Warn only when the issue actually mentions a project, so a+ // remote run without any referenced mention stays silent.+ const droppedProjectIds = [+ ...new Set(mentionedIds.filter((projectId) => projectId !== anchorProjectId)),+ ];+ return {+ additionalWorkspaces: [],+ warnings:+ droppedProjectIds.length > 0+ ? [+ "Referenced-project workspaces are available only on a local execution target or a confined sandbox target. This run uses a different remote execution target, so no referenced-project workspace was attached.",+ ]+ : [],+ failures: droppedProjectIds.map((projectId) => ({ projectId, reason: "staging" as const })),+ };+ }+ // Fall through: a confined sandbox target with the remote flag on resolves and authorizes the+ // referenced set exactly like a local target. The resolver is driver-agnostic; the confined+ // sandbox transport downstream stages each admitted tree into its own `project-<projectId>`+ // directory. The per-project `project:read` check below still runs against the run actor.}const referenced = await resolveRunReferencedProjects(issueId, anchorProjectId, {@@ -8192,20 +8252,27 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})agent: typeof agents.$inferSelect,context: Record<string, unknown>,previousSessionParams: Record<string, unknown> | null,- opts?: { useProjectWorkspace?: boolean | null; executionTargetIsRemote?: boolean },+ opts?: { useProjectWorkspace?: boolean | null; executionEnvironmentDriver?: string | null },): Promise<ResolvedWorkspaceForRun> {const anchor = await resolveAnchorWorkspaceForRun(agent, context, previousSessionParams, opts);if (!isMultiProjectWorkspaceSyncEnabled()) {return { ...anchor, additionalWorkspaces: [], referencedProjectFailures: [] };}+ // Derive the remote-transport facts from the selected environment driver. `executionTargetIsRemote`+ // decides whether the referenced set needs the remote path at all; `targetStagesConfined` decides+ // whether that remote target confines each staged tree (only the sandbox driver does). The remote+ // flag is the targeted kill switch; with it off, a remote run fails closed.+ const executionEnvironmentDriver = opts?.executionEnvironmentDriver ?? null;const issueId = readNonEmptyString(context.issueId) ?? readNonEmptyString(context.taskId);const { additionalWorkspaces, warnings, failures } = await resolveAdditionalRunWorkspaces(issueId,anchor.projectId,{enabled: true,- executionTargetIsRemote: opts?.executionTargetIsRemote ?? false,+ executionTargetIsRemote: isRemoteExecutionEnvironmentDriver(executionEnvironmentDriver),+ targetStagesConfined: isConfinedRemoteStagingDriver(executionEnvironmentDriver),+ remoteReferencedSyncEnabled: isMultiProjectWorkspaceSyncRemoteEnabled(),companyId: agent.companyId,actor: {type: "agent",@@ -13259,12 +13326,11 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})previousSessionParams,{useProjectWorkspace: requestedExecutionWorkspaceMode !== "agent_default",- // Referenced-project workspaces attach on a local execution target only. Gate their- // resolution on the selected environment driver so a remote run never resolves a- // referenced path it cannot reach. This never changes the anchor workspace.- executionTargetIsRemote: isRemoteExecutionEnvironmentDriver(- selectedEnvironmentForConfig?.driver,- ),+ // Thread the selected environment driver so run-workspace resolution can tell a local+ // target from a remote one, and a confined sandbox target from an unconfined remote+ // target. A remote run resolves referenced projects only for the confined sandbox+ // transport with the remote flag on. This never changes the anchor workspace.+ executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null,},),});@@ -14518,6 +14584,37 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}));}}+ // Reconcile the referenced-project set against the real remote staging outcome. A referenced+ // project can pass authorization and clone locally at run prep, then fail to stage into the+ // sandbox during execution. The run-prep observability above counts such a project as synced,+ // so emit a second, stage-time line that counts each staging failure as a first-class+ // `staging` failure. The synced set is the resolved referenced projects minus the ones that+ // failed to stage. A run with no staging failure stays silent, so the anchor-only and+ // fully-synced paths add no noise.+ const referencedProjectStagingFailures = adapterResult.referencedProjectStagingFailures ?? [];+ if (referencedProjectStagingFailures.length > 0) {+ const stagingFailedProjectIds = new Set(+ referencedProjectStagingFailures.map((failure) => failure.projectId),+ );+ const stagedProjectObservability = buildReferencedProjectRunObservability({+ syncedProjectIds: resolvedWorkspace.additionalWorkspaces+ .map((additional) => additional.projectId)+ .filter((projectId) => !stagingFailedProjectIds.has(projectId)),+ failures: referencedProjectStagingFailures.map((failure) => ({+ projectId: failure.projectId,+ reason: "staging" as const,+ })),+ });+ logger.info(+ {+ runId: run.id,+ companyId: agent.companyId,+ issueId: issueRef?.id ?? null,+ ...stagedProjectObservability,+ },+ "run referenced-project remote staging",+ );+ }const adapterManagedRuntimeServices = adapterResult.runtimeServices? await persistAdapterManagedRuntimeServices({db,diff --git a/server/src/services/workspace-realization.ts b/server/src/services/workspace-realization.tsindex 6049cfb36c..0c11f97d9d 100644--- a/server/src/services/workspace-realization.ts+++ b/server/src/services/workspace-realization.ts@@ -298,6 +298,14 @@ export function buildWorkspaceRealizationRecord(input: {};}+/**+ * Build the workspace-realization record from the run request. The server owns the record;+ * a driver realize handler (built-in or plugin) returns only a realized cwd and provider+ * metadata. Every `realizeWorkspace` exit must route through this helper, so the record carries+ * the referenced (mentioned) project sources in `additional`. The adapter reads `additional` to+ * stage each referenced tree into the target; a realize exit that returns a raw provider result+ * without this helper drops the mentioned projects.+ */export function buildWorkspaceRealizationRecordFromDriverInput(input: {environment: Environment;lease: EnvironmentLease;