The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
refactor(heartbeat): extract workspace preparation (#15807)
Dotta07fee85eafda
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex f58ef608b0..fd665cdb6d 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -1360,6 +1360,14 @@ lifecycle and ownership callbacks update its teardown state immediately, includiwhen preparation later throws. Public sandbox lifecycle and model fallback helpersremain re-exported by `heartbeat.ts`.+Workspace preparation is in `server/src/services/heartbeat/workspace-preparation.ts`.+`prepareHeartbeatWorkspace` resolves and provisions the workspace, persists its+metadata and issue binding, then acquires and realizes the execution environment.+It preserves the immutable workspace binding used by native recovery and records+preparation spans even when acquisition or realization fails. The executor retains+environment selection, dispatch gates, and terminal lease cleanup. Persistence+failures still clean newly created workspace artifacts before propagating.+Runtime execution is in `server/src/services/heartbeat/runtime-execution.ts`.`executeHeartbeatRuntime` dispatches a prepared native session or legacy adapter,captures usage, finalizes workspace operations, and releases gateway tokens anddiff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.tsindex 5517583e66..a0979025f2 100644--- a/server/src/__tests__/heartbeat-process-recovery.test.ts+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts@@ -8455,12 +8455,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {.set({ status: "paused" }).where(eq(agents.id, agentId));release();- if (next!)- await vi.waitFor(async () =>- expect((await heartbeat.getRun(next!.id))?.status).not.toBe(- "running",- ),- );+ // Wait for the released executor and its cleanup, not the default+ // one-second polling window. The next case must not inherit live work.+ await heartbeat.drainActiveRunExecutions();+ if (next!) expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");}},);@@ -12100,7 +12098,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {const legacyReviewBefore = withLegacyReview? await db.select().from(issues).where(eq(issues.id, legacyReviewId)): [];- mockAdapterExecute.mockImplementationOnce(async () => {+ let planOnlyRunId = "";+ mockAdapterExecute.mockImplementationOnce(async (input?: unknown) => {+ planOnlyRunId = (input as { runId: string }).runId;if (withLegacyReview) {// These pre-dispatch cancellations used to satisfy both the no-comment// and churn thresholds and suppress an otherwise valid continuation.@@ -12124,17 +12124,23 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {await heartbeat.reconcileStrandedAssignedIssues();await heartbeat.promoteDueScheduledRetries(new Date(Date.now() + 31_000));await heartbeat.resumeQueuedRuns();+ // The continuation is produced by completion; wait for that durable boundary.+ await heartbeat.drainActiveRunExecutions();- const livenessWake = await waitForValue(async () => {+ expect(planOnlyRunId).not.toBe("");+ const livenessWakes = await waitForValue(async () => {const rows = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.agentId, agentId));- return (- rows.find((row) => row.reason === "issue_disposition_repair") ?? null- );- });- expect(livenessWake).toBeTruthy();+ // Follow-up repairs can already exist. Match this plan-only result,+ // rather than relying on the database's unordered row order.+ const matching = rows.filter((row) => row.reason === "issue_disposition_repair"+ && row.payload?.retryOfRunId === planOnlyRunId);+ return matching.length > 0 ? matching : null;+ });+ expect(livenessWakes).toHaveLength(1);+ const livenessWake = livenessWakes![0]!;expect(livenessWake?.payload).toMatchObject({issueId,dispositionRepairAttempt: 1,@@ -12143,7 +12149,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {const sourceRunId = (livenessWake?.payload as Record<string, unknown> | null)?.retryOfRunId;- expect(sourceRunId).toBeTruthy();+ expect(sourceRunId).toBe(planOnlyRunId);const sourceRun = await db.select().from(heartbeatRuns)diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.tsindex ae86b02f87..576975ce90 100644--- a/server/src/services/heartbeat.ts+++ b/server/src/services/heartbeat.ts@@ -1,3 +1,4 @@+import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js";import { selectHeartbeatRuntime } from "./heartbeat/runtime-selection.js";import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js";@@ -145,19 +146,6 @@ import {readConfiguredModelFromAdapterConfig,resolveTaskSessionConfigFreshness,stripPaperclipSessionMetadataFromSessionParams,- resolveWorkspaceAfterLowTrustPreflight,- WorkspaceValidationFailure,- stripHostWorkspaceProvisionForLowTrustSandbox,- assertGitWorktreeBaseWorkspaceReady,- buildEffectiveRunWorkspaceConfigMetadata,- resolveExecutionWorkspaceConfigFreshness,- resolveExecutionWorkspaceReuseProvisioningPolicy,- provisionExecutionWorkspaceForFreshnessDecision,- mergeExecutionWorkspaceMetadataForPersistence,- resolveExecutionWorkspaceBranchOwnership,- reconcileReusedExecutionWorkspaceProjectWorkspaceId,- recordWorkspaceConfigFreshnessOperation,- prepareProjectRepositoryWorkspaces,resolveRuntimeSessionParamsForWorkspace,buildRunWorkspaceHints,buildReferencedProjectRunObservability,@@ -244,8 +232,6 @@ import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-idenimport { prepareConnectionInstructionDelivery } from "./connection-instructions.js";import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/assigned-mcp-tools.js";-import { resolvePaperclipInstanceRoot } from "../home-paths.js";-import {startupCancellationFence,} from "./native-runtime/native-cancellation-request.js";@@ -341,7 +327,6 @@ import {} from "node:crypto";import {and,- asc,desc,eq,exists,@@ -356,7 +341,6 @@ import {CHAT_PROVIDERS,ISSUE_DISPOSITION_REPAIR_RETRY_REASON,isEnvironmentDriverSupportedForAdapter,- type ExecutionWorkspace,} from "@paperclipai/shared";import {agents,@@ -372,14 +356,9 @@ import {projectWorkspaces,} from "@paperclipai/db";import { HttpError, notFound } from "../errors.js";-import {- getStartupTracer,-} from "../instrumentation.js";-import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js";-import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js";+import { logger } from "../middleware/logger.js";import {- createGitRemoteAuthProvider,resolveManagedGitHubIdentitySelection,scrubGitCredentialText,} from "./git-credentials.js";@@ -402,7 +381,6 @@ import {NativeCancellationPendingRecoveryError,NativeControllerDetachedForRestartError,prepareNativeWorkspaceSync,- readNativeWorkspaceSyncReference,type NativeRestartRecoveryClaim,resolveHeartbeatNativeRuntimeMode,} from "./native-runtime/index.js";@@ -444,11 +422,10 @@ import {} from "./native-runtime/native-runner-ownership.js";import {findNativeChatWorkspaceScope,- materializeNativeChatTaskRoot,nativeChatWorkspaceCwd,nativeChatWorkspaceMatches,} from "./native-runtime/native-chat-workspace.js";-import { materializeIsolatedTaskDirectory, shouldUseIsolatedTaskDirectory } from "./isolated-task-directory.js";+import { shouldUseIsolatedTaskDirectory } from "./isolated-task-directory.js";import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry";import { getTelemetryClient } from "../telemetry.js";import {@@ -481,21 +458,13 @@ import {buildWorkspaceReadyComment,buildWorkspaceReadyMetadata,buildWorkspaceReadyPresentation,- cleanupExecutionWorkspaceArtifacts,- ensurePersistedExecutionWorkspaceAvailable,ensureRuntimeServicesForRun,persistAdapterManagedRuntimeServices,- realizeExecutionWorkspace,releaseRuntimeServicesForRun,- type ExecutionWorkspaceInput,type RealizedExecutionWorkspace,type RuntimeServiceRef,} from "./workspace-runtime.js";-import {- readManagedWorktreeInstanceOwnership,- WORKTREE_INSTANCE_ROOT_METADATA_KEY,-} from "./workspace-instance-cleanup.js";import { issueService } from "./issues.js";import {failRunnerGoalAction,@@ -521,9 +490,7 @@ import {import {isProcessGroupAlive,} from "./local-service-supervisor.js";-import {- isRuntimeOwnedGitBranch,-} from "./execution-workspace-branch-ownership.js";+import {HEARTBEAT_RUN_SCRATCH_MARKER,buildHeartbeatRunScratchEnv,@@ -535,10 +502,8 @@ import {applyDefaultIsolatedExecutionWorkspacePolicy,buildExecutionWorkspaceAdapterConfig,gateProjectExecutionWorkspacePolicy,- issueExecutionWorkspaceModeForPersistedWorkspace,parseIssueExecutionWorkspaceSettings,parseProjectExecutionWorkspacePolicy,- resolveEffectiveWorkspaceStrategyType,resolveExecutionWorkspaceEnvironmentId,resolveExecutionWorkspaceMode,resolveSharedWorkspaceConcurrency,@@ -549,7 +514,6 @@ import {resolveWorktreeRunExecutionActivation,} from "./instance-settings.js";import {- evaluateExecutionAllowlist,isExecutionForcedToKubernetes,} from "./execution-allowlist.js";@@ -3692,862 +3656,78 @@ export function heartbeatService(if (handoff) context.paperclipFreshSessionHandoffMarkdown = handoff;}const {- selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,- workspace: resolvedWorkspace,- } = await resolveWorkspaceAfterLowTrustPreflight({- db,- trustPreset,- isolatedWorkspacesEnabled,- effectiveExecutionWorkspaceMode,- issue: issueRef- ? {- companyId: agent.companyId,- id: issueRef.id,- projectId: issueRef.projectId,- }- : null,- resolveSelectedEnvironmentDriver: async () => {- const preflightEnvironment = await envOrchestrator.resolveEnvironment(- {- companyId: agent.companyId,- selectedEnvironmentId,- localEnvironmentId: localEnvironment.id,- },- );- return preflightEnvironment.driver;- },- resolveWorkspace: async () => {- if (isDotRun) {- // This is private controller storage, never a provider filesystem.- // v6 projects workspace.access=none and cwd=null to Dot.- const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "dot-controllers", agent.companyId, run.id);- await fs.mkdir(cwd, { recursive: true, mode: 0o700 });- return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null,- workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] };- }- if (useIsolatedTaskDirectory && issueRef) {- const cwd = await materializeIsolatedTaskDirectory({- companyId: agent.companyId,- issueId: issueRef.id,- });- if (reusableExistingExecutionWorkspace && (- reusableExistingExecutionWorkspace.companyId !== agent.companyId ||- reusableExistingExecutionWorkspace.projectId !== issueRef.projectId ||- reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id ||- reusableExistingExecutionWorkspace.mode !== "isolated_workspace" ||- reusableExistingExecutionWorkspace.strategyType !== "project_primary" ||- reusableExistingExecutionWorkspace.cwd !== cwd- )) {- throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", {- workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id },- });- }- return resolveWorkspaceForRun(agent, context, previousSessionParams, {- executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null,- anchorWorkspace: {- cwd,- source: "task_session",- projectId: issueRef.projectId,- workspaceId: null,- repoUrl: null,- repoRef: null,- workspaceHints: [],- warnings: [],- baseCwdFallback: false,- materializationFailures: [],- },- });- }- if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) {- const cwd = await materializeNativeChatTaskRoot(- nativeChatWorkspaceScope,- );- return {- cwd,- source: "task_session" as const,- projectId: null,- workspaceId: null,- repoUrl: null,- repoRef: null,- workspaceHints: [],- warnings: [],- baseCwdFallback: false,- materializationFailures: [],- additionalWorkspaces: [],- referencedProjectFailures: [],- };- }- const workspace = await resolveWorkspaceForRun(- agent,- context,- previousSessionParams,- {- useProjectWorkspace:- requestedExecutionWorkspaceMode !== "agent_default",- anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"- ? await resolveReusedGitWorkspaceAnchor({- agent,- workspace: reusableExistingExecutionWorkspace,- responsibleUserId,- immutableNativeBinding: Boolean(nativeRecoveryExecutionWorkspaceId),- projectId: nativeRecoveryExecutionWorkspaceId- ? reusableExistingExecutionWorkspace.projectId- : issueRef?.projectId ?? readNonEmptyString(context.projectId),- explicitProjectWorkspaceId: nativeRecoveryExecutionWorkspaceId- ? reusableExistingExecutionWorkspace.projectWorkspaceId- : readNonEmptyString(context.projectWorkspaceId),- issueId,- runId: run.id,- })- : undefined,- // Thread the selected environment driver so run-workspace resolution can tell a local- // target from a remote one, and a confined sandbox target from an unconfined remote- // target. A remote run resolves referenced projects only for the confined sandbox- // transport with the remote flag on. This never changes the anchor workspace.- executionEnvironmentDriver:- selectedEnvironmentForConfig?.driver ?? null,- },- );- // Additional referenced projects are a separate trusted Board- // capability, not extra readable roots for an external conversation.- return nativeChatWorkspaceScope- ? {- ...workspace,- additionalWorkspaces: [],- referencedProjectFailures: [],- }- : workspace;+ resolvedWorkspace,+ hostExecutionWorkspaceConfig,+ latestWorkspaceConfigMetadata,+ workspaceConfigFreshness,+ workspaceOperationRecorder,+ executionWorkspace,+ reusedExecutionWorkspace,+ resolvedWorkspaceReusePolicy,+ persistedExecutionWorkspace,+ projectRepositoryPaths,+ environmentAcquireStartedAtMs,+ selectedEnvironment,+ activeEnvironmentLease,+ realizationResult,+ environmentRealizeEndedAtMs,+ workspaceRealization,+ executionTarget,+ workspaceRestoreSource,+ } = await prepareHeartbeatWorkspace(db, {+ run, agent, controllerLease, nativeRunnerPreparationSpans,+ task: {+ issueRef,+ issueId,+ context,+ executionProjectId,+ responsibleUserId,+ previousSessionParams,},- });- const hostExecutionWorkspaceConfig = isDotRun ? {} :- stripHostWorkspaceProvisionForLowTrustSandbox({- config: mergedConfig,+ policy: {trustPreset,- selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,- });- const executionWorkspaceBase = {- baseCwd: resolvedWorkspace.cwd,- source: resolvedWorkspace.source,- projectId: resolvedWorkspace.projectId,- workspaceId: resolvedWorkspace.workspaceId,- repoUrl: resolvedWorkspace.repoUrl,- repoRef: resolvedWorkspace.repoRef,- additionalWorkspaces: resolvedWorkspace.additionalWorkspaces,- } satisfies ExecutionWorkspaceInput;- await assertGitWorktreeBaseWorkspaceReady({- requestedExecutionWorkspaceMode,- config: hostExecutionWorkspaceConfig,- issue: issueRef,- base: executionWorkspaceBase,- anchor: {- baseCwdFallback: resolvedWorkspace.baseCwdFallback,- materializationFailures: resolvedWorkspace.materializationFailures,- localPathOnlyWorkspace: resolvedWorkspace.localPathOnlyWorkspace,+ isolatedWorkspacesEnabled,+ effectiveExecutionWorkspaceMode,+ requestedExecutionWorkspaceMode,+ useIsolatedTaskDirectory,+ nativeChatWorkspaceScope,+ projectExecutionWorkspacePolicy,+ issueExecutionWorkspaceSettings,+ environmentExecutionWorkspaceSettings,+ executionPolicy,},- });- const workspaceStrategyForFingerprint = parseObject(- hostExecutionWorkspaceConfig.workspaceStrategy,- );- const workspaceStrategyFingerprintValue =- Object.keys(workspaceStrategyForFingerprint).length > 0- ? workspaceStrategyForFingerprint- : null;- const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType(- requestedExecutionWorkspaceMode,- hostExecutionWorkspaceConfig,- );- const selectedEnvironmentConfigForFingerprint = parseObject(- selectedEnvironmentForConfig?.config,- );- const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig- ? {- selectionSource: environmentResolution.source,- selectedEnvironmentId,- driver: selectedEnvironmentForConfig.driver,- provider: readNonEmptyString(- selectedEnvironmentConfigForFingerprint.provider,- ),- config: selectedEnvironmentForConfig.config,- configRevisionAt:- selectedEnvironmentForConfig.updatedAt instanceof Date- ? selectedEnvironmentForConfig.updatedAt.toISOString()- : (selectedEnvironmentForConfig.updatedAt ?? null),- executionPolicy,- }- : null;- const workspaceRealizationFingerprint = {- environmentDriver: selectedEnvironmentForConfig?.driver ?? null,- environmentProvider: readNonEmptyString(- selectedEnvironmentConfigForFingerprint.provider,- ),- trustPreset: trustPreset.kind,- lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver,- };- const workspaceFreshnessSource = resolvedWorkspace.freshnessSource ?? executionWorkspaceBase;- const latestWorkspaceConfigMetadata =- buildEffectiveRunWorkspaceConfigMetadata({- mode: requestedExecutionWorkspaceMode,- projectId: workspaceFreshnessSource.projectId,- projectWorkspaceId: workspaceFreshnessSource.workspaceId,- strategyType: latestWorkspaceStrategyType,- workspaceStrategy: workspaceStrategyFingerprintValue,- repoUrl: workspaceFreshnessSource.repoUrl,- repoRef:- readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ??- workspaceFreshnessSource.repoRef,+ environment: {+ selectedEnvironmentId,+ localEnvironment,+ selectedEnvironmentForConfig,+ environmentResolution,+ resolvedInstanceSettings,+ },+ config: {+ mergedConfig,configSnapshot,- environment: workspaceEnvironmentFingerprint,- realization: workspaceRealizationFingerprint,secretManifest,- });- const inferredExistingWorkspaceConfigMetadata =- reusableExistingExecutionWorkspace- ? buildEffectiveRunWorkspaceConfigMetadata({- mode: issueExecutionWorkspaceModeForPersistedWorkspace(- reusableExistingExecutionWorkspace.mode,- ),- projectId: reusableExistingExecutionWorkspace.projectId,- projectWorkspaceId:- reusableExistingExecutionWorkspace.projectWorkspaceId,- strategyType: reusableExistingExecutionWorkspace.strategyType,- workspaceStrategy: workspaceStrategyFingerprintValue- ? {- ...workspaceStrategyFingerprintValue,- type: reusableExistingExecutionWorkspace.strategyType,- ...(reusableExistingExecutionWorkspace.baseRef- ? { baseRef: reusableExistingExecutionWorkspace.baseRef }- : {}),- }- : { type: reusableExistingExecutionWorkspace.strategyType },- repoUrl: reusableExistingExecutionWorkspace.repoUrl,- repoRef: reusableExistingExecutionWorkspace.baseRef,- configSnapshot: reusableExistingExecutionWorkspace.config,- environment: workspaceEnvironmentFingerprint,- realization: workspaceRealizationFingerprint,- secretManifest,- evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt,- })- : null;- const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness(- {- hasExistingWorkspace:- requestedShouldReuseExisting &&- Boolean(reusableExistingExecutionWorkspace),- existingWorkspaceMetadata:- reusableExistingExecutionWorkspace?.metadata ?? null,- inferredMetadata: inferredExistingWorkspaceConfigMetadata,- nextMetadata: latestWorkspaceConfigMetadata,},- );- const workspaceReuseProvisioningPolicy =- resolveExecutionWorkspaceReuseProvisioningPolicy({+ reuse: {requestedShouldReuseExisting,- workspaceConfigFreshness,- });- const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({- companyId: agent.companyId,- heartbeatRunId: run.id,- executionWorkspaceId:- workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace- ? workspaceReuseRequest.requestedExecutionWorkspaceId- : null,- issueId,- });- // The run-scoped provider resolves the active identity at each Git operation,- // including base-ref refreshes, workspace realization, and restore.- const workspaceGitAuthProvider = createGitRemoteAuthProvider(- db,- agent.companyId,- {- issueId,- heartbeatRunId: run.id,- responsibleUserId: run.responsibleUserId,- agentId: agent.id,+ existingExecutionWorkspace,+ reusableExistingExecutionWorkspace,+ workspaceReuseRequest,+ nativeRecoveryExecutionWorkspaceId,+ persistedNativeExecutionInput,+ isDotRun,+ runOptions,},- );- const {- executionWorkspace,- reusedExecutionWorkspace,- policy: resolvedWorkspaceReusePolicy,- } = isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(- {- requestedShouldReuseExisting,- existingExecutionWorkspaceId:- workspaceReuseRequest.requestedExecutionWorkspaceId,- issueRef,- runId: run.id,- workspaceConfigFreshness,- restoreExistingWorkspace: reusableExistingExecutionWorkspace- ? () =>- ensurePersistedExecutionWorkspaceAvailable({- db,- base: executionWorkspaceBase,- workspace: {- id: reusableExistingExecutionWorkspace.id,- mode: reusableExistingExecutionWorkspace.mode,- strategyType:- reusableExistingExecutionWorkspace.strategyType,- cwd: reusableExistingExecutionWorkspace.cwd,- providerRef: reusableExistingExecutionWorkspace.providerRef,- projectId: reusableExistingExecutionWorkspace.projectId,- projectWorkspaceId:- reusableExistingExecutionWorkspace.projectWorkspaceId,- repoUrl: reusableExistingExecutionWorkspace.repoUrl,- baseRef: reusableExistingExecutionWorkspace.baseRef,- branchName: reusableExistingExecutionWorkspace.branchName,- metadata:- reusableExistingExecutionWorkspace.metadata as Record<- string,- unknown- > | null,- config: {- provisionCommand:- configSnapshot?.provisionCommand ??- reusableExistingExecutionWorkspace.config- ?.provisionCommand ??- projectExecutionWorkspacePolicy?.workspaceStrategy- ?.provisionCommand ??- null,- runtimeProvisionCommand:- configSnapshot?.runtimeProvisionCommand ??- reusableExistingExecutionWorkspace.config- ?.runtimeProvisionCommand ??- projectExecutionWorkspacePolicy?.workspaceStrategy- ?.runtimeProvisionCommand ??- null,- },- },- issue: issueRef,- agent: {- id: agent.id,- name: agent.name,- companyId: agent.companyId,- },- heartbeatRunId: run.id,- enableWorkspaceBranchReconcileForward:- resolvedInstanceSettings.experimental- .enableWorkspaceBranchReconcileForward,- enableWorkspaceDirtyQuarantineRepair:- resolvedInstanceSettings.experimental- .enableWorkspaceDirtyQuarantineRepair,- recorder: workspaceOperationRecorder,- resolveGitAuth: workspaceGitAuthProvider,- })- : null,- realizeWorkspace: () =>- realizeExecutionWorkspace({- db,- base: executionWorkspaceBase,- config: hostExecutionWorkspaceConfig,- issue: issueRef,- agent: {- id: agent.id,- name: agent.name,- companyId: agent.companyId,- },- recordedBranchOwnership:- existingExecutionWorkspace?.status !== "archived" &&- existingExecutionWorkspace?.branchName- ? {- branchName: existingExecutionWorkspace.branchName,- createdByRuntime: isRuntimeOwnedGitBranch(- existingExecutionWorkspace.metadata,- ),- }- : null,- heartbeatRunId: run.id,- enableWorkspaceBranchReconcileForward:- resolvedInstanceSettings.experimental- .enableWorkspaceBranchReconcileForward,- enableWorkspaceDirtyQuarantineRepair:- resolvedInstanceSettings.experimental- .enableWorkspaceDirtyQuarantineRepair,- recorder: workspaceOperationRecorder,- resolveGitAuth: workspaceGitAuthProvider,- }),+ services: {+ envOrchestrator,+ executionWorkspacesSvc,+ workspaceOperationsSvc,+ issuesSvc,+ resolveWorkspaceForRun,+ resolveReusedGitWorkspaceAnchor,+ appendRunEvent,},- );- const resolvedProjectId =- executionWorkspace.projectId ??- issueRef?.projectId ??- executionProjectId ??- null;- const resolvedProjectWorkspaceId =- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"- ? reusableExistingExecutionWorkspace.projectWorkspaceId- : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;- let persistedExecutionWorkspace: ExecutionWorkspace | null = null;- let issueExecutionWorkspaceIdForRun =- issueRef?.executionWorkspaceId ?? null;- let issueProjectWorkspaceIdForRun = issueRef?.projectWorkspaceId ?? null;- let issueExecutionWorkspacePreferenceForRun =- issueRef?.executionWorkspacePreference ?? null;- let issueExecutionWorkspaceModeForRun =- issueExecutionWorkspaceSettings?.mode ?? null;- const warmReusableExecutionWorkspace =- selectedEnvironmentForConfig?.driver === "sandbox" &&- selectedEnvironmentConfigForFingerprint.reuseLease === true &&- selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm";- // Native provider checkpoints bind to the workspace row, including ordinary- // local shared workspaces. Persist that binding independently of the opt-in- // isolated-workspace UI, just as warm sandbox continuity already does.- const nativeSharedWorkspace = agent.adapterType === "paperclip_runner" &&- requestedExecutionWorkspaceMode === "shared_workspace";- const bindIssueToPersistedExecutionWorkspace = async (- workspace: ExecutionWorkspace | null,- ) => {- if (!issueId || !workspace || nativeRecoveryExecutionWorkspaceId) {- return;- }- const nextIssueWorkspaceMode =- issueExecutionWorkspaceModeForPersistedWorkspace(workspace.mode) ??- "agent_default";- const shouldSwitchIssueToExistingWorkspace =- issueRef?.executionWorkspacePreference === "reuse_existing" ||- requestedExecutionWorkspaceMode === "isolated_workspace" ||- requestedExecutionWorkspaceMode === "operator_branch" ||- warmReusableExecutionWorkspace || nativeSharedWorkspace;- const nextIssuePatch: Record<string, unknown> = {};- if (issueExecutionWorkspaceIdForRun !== workspace.id) {- nextIssuePatch.executionWorkspaceId = workspace.id;- }- if (- resolvedProjectWorkspaceId &&- issueProjectWorkspaceIdForRun !== resolvedProjectWorkspaceId- ) {- nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;- }- if (- shouldSwitchIssueToExistingWorkspace &&- (issueExecutionWorkspacePreferenceForRun !== "reuse_existing" ||- issueExecutionWorkspaceModeForRun !== nextIssueWorkspaceMode)- ) {- nextIssuePatch.executionWorkspacePreference = "reuse_existing";- nextIssuePatch.executionWorkspaceSettings = {- ...(issueExecutionWorkspaceSettings ?? {}),- mode: nextIssueWorkspaceMode,- };- }- if (Object.keys(nextIssuePatch).length > 0) {- await issuesSvc.update(- issueId,- { ...nextIssuePatch, companyGuard: agent.companyId },- db,- undefined,- undefined,- { bindRuntimeSharedWorkspace: (warmReusableExecutionWorkspace || nativeSharedWorkspace) && workspace.mode === "shared_workspace" },- );- issueExecutionWorkspaceIdForRun = workspace.id;- issueProjectWorkspaceIdForRun =- resolvedProjectWorkspaceId ?? issueProjectWorkspaceIdForRun;- if (shouldSwitchIssueToExistingWorkspace) {- issueExecutionWorkspacePreferenceForRun = "reuse_existing";- issueExecutionWorkspaceModeForRun = nextIssueWorkspaceMode;- }- }- };- const baseExecutionWorkspaceMetadata =- mergeExecutionWorkspaceMetadataForPersistence({- existingMetadata:- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace- ? (reusableExistingExecutionWorkspace?.metadata ?? null)- : null,- source: executionWorkspace.source,- // Attaching a new worktree to a pre-existing branch reports a fresh- // workspace, but must not make cleanup own the operator's branch.- createdByRuntime:- resolveExecutionWorkspaceBranchOwnership(executionWorkspace),- strategyType: executionWorkspace.strategy,- configSnapshot,- shouldReuseExisting:- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace,- shouldRefreshConfigSnapshot:- resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,- workspaceConfigMetadata:- resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata- ? latestWorkspaceConfigMetadata- : null,- baseRef: executionWorkspace.repoRef,- baseRefSha: executionWorkspace.baseRefSha ?? null,- });- let persistedWorktreeInstanceRoot =- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&- typeof reusableExistingExecutionWorkspace?.metadata?.[- WORKTREE_INSTANCE_ROOT_METADATA_KEY- ] === "string"- ? reusableExistingExecutionWorkspace.metadata[- WORKTREE_INSTANCE_ROOT_METADATA_KEY- ]- : null;- if (- !persistedWorktreeInstanceRoot &&- executionWorkspace.strategy === "git_worktree" &&- executionWorkspace.worktreePath- ) {- try {- persistedWorktreeInstanceRoot =- (- await readManagedWorktreeInstanceOwnership(- executionWorkspace.worktreePath,- )- )?.instanceRoot ?? null;- } catch (error) {- logger.warn(- {- runId: run.id,- issueId,- executionWorkspaceCwd: executionWorkspace.cwd,- error: error instanceof Error ? error.message : String(error),- },- "Could not record managed worktree instance ownership",- );- }- }- const nextExecutionWorkspaceMetadata = {- ...baseExecutionWorkspaceMetadata,- ...(persistedWorktreeInstanceRoot- ? {- [WORKTREE_INSTANCE_ROOT_METADATA_KEY]:- persistedWorktreeInstanceRoot,- }- : {}),- };- const pendingForwardBranchReconcile =- executionWorkspace.pendingForwardBranchReconcile ?? null;- const branchNameForInitialPersistence =- pendingForwardBranchReconcile?.recordedBranchName ??- executionWorkspace.branchName;- try {- persistedExecutionWorkspace =- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&- reusableExistingExecutionWorkspace- ? await executionWorkspacesSvc.update(- reusableExistingExecutionWorkspace.id,- {- cwd: executionWorkspace.cwd,- repoUrl: executionWorkspace.repoUrl,- baseRef: executionWorkspace.repoRef,- branchName: branchNameForInitialPersistence,- providerType:- executionWorkspace.strategy === "git_worktree"- ? "git_worktree"- : "local_fs",- providerRef: executionWorkspace.worktreePath,- status: "active",- lastUsedAt: new Date(),- metadata: nextExecutionWorkspaceMetadata,- projectWorkspaceId:- reconcileReusedExecutionWorkspaceProjectWorkspaceId(- reusableExistingExecutionWorkspace.projectWorkspaceId,- resolvedProjectWorkspaceId,- ),- },- )- : resolvedProjectId- ? await executionWorkspacesSvc.create({- companyId: agent.companyId,- projectId: resolvedProjectId,- projectWorkspaceId: resolvedProjectWorkspaceId,- sourceIssueId: issueRef?.id ?? null,- mode:- requestedExecutionWorkspaceMode === "isolated_workspace"- ? "isolated_workspace"- : requestedExecutionWorkspaceMode === "operator_branch"- ? "operator_branch"- : requestedExecutionWorkspaceMode === "agent_default"- ? "adapter_managed"- : "shared_workspace",- strategyType:- executionWorkspace.strategy === "git_worktree"- ? "git_worktree"- : "project_primary",- name:- branchNameForInitialPersistence ??- issueRef?.identifier ??- `workspace-${agent.id.slice(0, 8)}`,- status: "active",- cwd: executionWorkspace.cwd,- repoUrl: executionWorkspace.repoUrl,- baseRef: executionWorkspace.repoRef,- branchName: branchNameForInitialPersistence,- providerType:- executionWorkspace.strategy === "git_worktree"- ? "git_worktree"- : "local_fs",- providerRef: executionWorkspace.worktreePath,- lastUsedAt: new Date(),- openedAt: new Date(),- metadata: nextExecutionWorkspaceMetadata,- })- : null;- } catch (error) {- if (executionWorkspace.created) {- try {- await cleanupExecutionWorkspaceArtifacts({- workspace: {- id:- reusableExistingExecutionWorkspace?.id ??- workspaceReuseRequest.requestedExecutionWorkspaceId ??- `transient-${run.id}`,- cwd: executionWorkspace.cwd,- providerType:- executionWorkspace.strategy === "git_worktree"- ? "git_worktree"- : "local_fs",- providerRef: executionWorkspace.worktreePath,- branchName: executionWorkspace.branchName,- repoUrl: executionWorkspace.repoUrl,- baseRef: executionWorkspace.repoRef,- projectId: resolvedProjectId,- projectWorkspaceId: resolvedProjectWorkspaceId,- sourceIssueId: issueRef?.id ?? null,- metadata: nextExecutionWorkspaceMetadata,- },- projectWorkspace: {- cwd: resolvedWorkspace.cwd,- cleanupCommand: null,- },- cleanupCommand: configSnapshot?.cleanupCommand ?? null,- teardownCommand:- configSnapshot?.teardownCommand ??- projectExecutionWorkspacePolicy?.workspaceStrategy- ?.teardownCommand ??- null,- recorder: workspaceOperationRecorder,- });- } catch (cleanupError) {- logger.warn(- {- runId: run.id,- issueId,- executionWorkspaceCwd: executionWorkspace.cwd,- cleanupError:- cleanupError instanceof Error- ? cleanupError.message- : String(cleanupError),- },- "Failed to cleanup realized execution workspace after persistence failure",- );- }- }- throw error;- }- await workspaceOperationRecorder.attachExecutionWorkspaceId(- persistedExecutionWorkspace?.id ?? null,- );- await recordWorkspaceConfigFreshnessOperation({- recorder: workspaceOperationRecorder,- runId: run.id,- decision: workspaceConfigFreshness,- hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace),- reuseRequested: requestedShouldReuseExisting,- workspaceReused: Boolean(reusedExecutionWorkspace),- configSnapshotRefreshed:- resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,- previousWorkspaceId:- workspaceReuseRequest.requestedExecutionWorkspaceId,- activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,- });- if (- reusableExistingExecutionWorkspace &&- persistedExecutionWorkspace &&- reusableExistingExecutionWorkspace.id !==- persistedExecutionWorkspace.id &&- reusableExistingExecutionWorkspace.status === "active"- ) {- await executionWorkspacesSvc.update(- reusableExistingExecutionWorkspace.id,- {- status: "idle",- cleanupReason: null,- },- );- }- await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);- const projectRepositoryPaths: string[] = [];- if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) {- const repositoryRows = await db.select().from(projectWorkspaces).where(and(- eq(projectWorkspaces.companyId, agent.companyId),- eq(projectWorkspaces.projectId, executionWorkspace.projectId),- )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id));- const repositories = await prepareProjectRepositoryWorkspaces({- cwd: executionWorkspace.cwd,- anchorRepoUrl: executionWorkspace.repoUrl,- workspaces: repositoryRows,- resolveGitAuth: workspaceGitAuthProvider,- });- const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd]));- projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd)));- if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd);- resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({- ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd,- }));- }- if (persistedExecutionWorkspace) {- context.executionWorkspaceId = persistedExecutionWorkspace.id;- await db- .update(heartbeatRuns)- .set({- contextSnapshot: context,- updatedAt: new Date(),- })- .where(eq(heartbeatRuns.id, run.id));- }- const environmentAcquireStartedAtMs = Date.now();- let acquiredEnvironment: Awaited<- ReturnType<typeof envOrchestrator.acquireForRun>- >;- try {- await controllerLease.assertOwned();- const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner"- ? runOptions.nativeRestartRecovery : null;- const recoveryWorkspace = remoteRecovery- ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null;- if (remoteRecovery && (!recoveryWorkspace || remoteRecovery.runId !== run.id ||- recoveryWorkspace.providerLeaseId !== remoteRecovery.remote.providerLeaseId ||- recoveryWorkspace.remoteCwd !== remoteRecovery.remote.remoteCwd)) {- throw new Error("native_remote_recovery_lease_mismatch");- }- acquiredEnvironment = await envOrchestrator.acquireForRun({- companyId: agent.companyId,- selectedEnvironmentId,- localEnvironmentId: localEnvironment.id,- adapterType: agent.adapterType,- adapterConfig: parseObject(agent.adapterConfig),- admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode,- issueId: issueId ?? null,- heartbeatRunId: run.id,- agentId: agent.id,- persistedExecutionWorkspace,- executionWorkspaceSettings: environmentExecutionWorkspaceSettings,- ...(remoteRecovery && recoveryWorkspace ? { reattachRemoteLease: {- leaseId: recoveryWorkspace.leaseId,- providerLeaseId: remoteRecovery.remote.providerLeaseId,- remoteCwd: remoteRecovery.remote.remoteCwd,- } } : {}),- });- await controllerLease.assertOwned();- nativeRunnerPreparationSpans.push({- name: "environment.acquire",- parentName: "task.run",- startedAtMs: environmentAcquireStartedAtMs,- endedAtMs: Date.now(),- attributes: { adapter: agent.adapterType },- });- } catch (error) {- nativeRunnerPreparationSpans.push({- name: "environment.acquire",- parentName: "task.run",- startedAtMs: environmentAcquireStartedAtMs,- endedAtMs: Date.now(),- outcome: "failed",- attributes: { adapter: agent.adapterType },- });- throw error;- }- const selectedEnvironment = acquiredEnvironment.environment;- // Defense-in-depth: re-check the actually-acquired environment against the- // execution allowlist. Even if selection were bypassed, a denied (local/ssh/- // non-k8s) environment FAILS the run here rather than executing untrusted.- const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, {- driver: selectedEnvironment.driver,- provider:- typeof selectedEnvironment.config?.provider === "string"- ? selectedEnvironment.config.provider- : null,});- if (!allowlistDecision.allowed) {- logger.error(- {- runId: run.id,- issueId,- agentId: agent.id,- environmentId: selectedEnvironment.id,- deniedDriver: allowlistDecision.deniedDriver,- deniedProvider: allowlistDecision.deniedProvider,- },- "Execution allowlist denied the resolved environment; failing run",- );- throw new Error(allowlistDecision.reason);- }- let activeEnvironmentLease = {- environment: acquiredEnvironment.environment,- lease: acquiredEnvironment.lease,- leaseContext: acquiredEnvironment.leaseContext,- };- const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder(- {- tracer: getStartupTracer(),- incrementCounter: (metric) => {- void incrementToolRuntimeMetricCounter(db, {- companyId: run.companyId,- metric,- }).catch(() => {});- },- emitTransportEvent: (event) => {- void (async () => {- await appendRunEvent(run, {- eventType: event.name,- stream: "system",- level: event.dimensions.outcome === "error" ? "warn" : "info",- payload: { ...event.dimensions },- });- })().catch(() => {});- },- },- );- const environmentRealizeStartedAtMs = Date.now();- let realizationResult: Awaited<- ReturnType<typeof envOrchestrator.realizeForRun>- >;- try {- realizationResult = await envOrchestrator.realizeForRun({- environment: selectedEnvironment,- lease: activeEnvironmentLease.lease,- adapterType: agent.adapterType,- companyId: agent.companyId,- issueId: issueId ?? null,- heartbeatRunId: run.id,- executionWorkspace,- effectiveExecutionWorkspaceMode,- persistedExecutionWorkspace,- duplexObservabilityRecorder,- });- nativeRunnerPreparationSpans.push({- name: "environment.workspace.realize",- parentName: "task.run",- startedAtMs: environmentRealizeStartedAtMs,- endedAtMs: Date.now(),- attributes: { driver: selectedEnvironment.driver },- });- } catch (error) {- nativeRunnerPreparationSpans.push({- name: "environment.workspace.realize",- parentName: "task.run",- startedAtMs: environmentRealizeStartedAtMs,- endedAtMs: Date.now(),- outcome: "failed",- attributes: { driver: selectedEnvironment.driver },- });- throw error;- }- const environmentRealizeEndedAtMs = Date.now();- activeEnvironmentLease = {- ...activeEnvironmentLease,- lease: realizationResult.lease,- };- persistedExecutionWorkspace =- realizationResult.persistedExecutionWorkspace;- // A sandbox realization may materialize or replace the durable workspace- // after the host-side provisioning boundary above. Bind that final ID to- // the issue before dispatch so warm turns reuse the exact same workspace- // and lease scope instead of silently creating a per-run replacement.- await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);- const workspaceRealization = realizationResult.workspaceRealization;- const executionTarget = realizationResult.executionTarget;- // Preserve the host-owned source before adapter context can share lease- // metadata. A later copy-back failure must not adopt a rebound source.- const workspaceRestoreSource = structuredClone(realizationResult.lease);let instructionCopy: Awaited<ReturnType<typeof instructionCopies.prepare>> = null;let instructionSave: Record<string, unknown> | null = null;const instructionPreparationKey = createHash("sha256").update(JSON.stringify({diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.tsnew file mode 100644index 0000000000..c251f4e02a--- /dev/null+++ b/server/src/services/heartbeat/workspace-preparation.test.ts@@ -0,0 +1,236 @@+import { execFile as execFileCallback } from "node:child_process";+import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { promisify } from "node:util";+import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issues, projects, projectWorkspaces, type Db } from "@paperclipai/db";+import { eq, sql } from "drizzle-orm";+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";+import { environmentRunOrchestrator } from "../environment-run-orchestrator.js";+import { environmentService } from "../environments.js";+import { executionWorkspaceService } from "../execution-workspaces.js";+import { instanceSettingsService } from "../instance-settings.js";+import { issueService } from "../issues.js";+import { workspaceOperationService } from "../workspace-operations.js";+import { createHeartbeatWorkspaceResolver, resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js";+import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";++const execFile = promisify(execFileCallback);+const support = await getEmbeddedPostgresTestSupport();++describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", () => {+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;+ let db: Db;+ let home: string;++ beforeAll(async () => {+ home = await mkdtemp(join(tmpdir(), "paperclip-workspace-preparation-"));+ vi.stubEnv("PAPERCLIP_HOME", home);+ database = await startEmbeddedPostgresTestDatabase("paperclip-workspace-preparation");+ db = createDb(database.connectionString);+ await db.execute(sql`set client_min_messages = warning`);+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);+ afterAll(async () => {+ await db?.$client.end();+ await database?.cleanup();+ vi.unstubAllEnvs();+ if (home) await rm(home, { recursive: true, force: true });+ });+ beforeEach(async () => {+ vi.restoreAllMocks();+ await db.execute(sql`truncate table companies restart identity cascade`);+ await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });+ });++ async function fixture(isolated = false) {+ const [company] = await db.insert(companies).values({ name: "Workspace preparation", issuePrefix: "WSP" }).returning();+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Workspace agent", status: "running", adapterType: "process" }).returning();+ const [project] = await db.insert(projects).values({ companyId: company.id, name: "Source project" }).returning();+ const source = join(home, project.id);+ await mkdir(source);+ await execFile("git", ["init", "--initial-branch=main", source]);+ await execFile("git", ["-C", source, "config", "user.email", "[email protected]"]);+ await execFile("git", ["-C", source, "config", "user.name", "Workspace test"]);+ await writeFile(join(source, "README.md"), "source content\n");+ await execFile("git", ["-C", source, "add", "."]);+ await execFile("git", ["-C", source, "commit", "-m", "Source"]);+ const [projectWorkspace] = await db.insert(projectWorkspaces).values({ companyId: company.id, projectId: project.id, name: "Source", cwd: source, isPrimary: true }).returning();+ const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Prepare this workspace", identifier: "WSP-1", projectId: project.id, projectWorkspaceId: projectWorkspace.id, assigneeAgentId: agent.id }).returning();+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();+ const localEnvironment = await environmentService(db).ensureLocalEnvironment(company.id);+ const envOrchestrator = environmentRunOrchestrator(db);+ const resolver = createHeartbeatWorkspaceResolver(db);+ const input: HeartbeatWorkspacePreparationInput = {+ run, agent,+ task: { issueRef: issue, issueId: issue.id, context: { issueId: issue.id, projectId: project.id }, executionProjectId: project.id, responsibleUserId: null, previousSessionParams: null },+ policy: {+ trustPreset: { kind: "standard", preset: "standard", boundary: null, sourcePresets: {} },+ isolatedWorkspacesEnabled: true,+ effectiveExecutionWorkspaceMode: isolated ? "isolated_workspace" : "shared_workspace",+ requestedExecutionWorkspaceMode: isolated ? "isolated_workspace" : "shared_workspace",+ useIsolatedTaskDirectory: false, nativeChatWorkspaceScope: null,+ projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null, environmentExecutionWorkspaceSettings: null,+ executionPolicy: { executionMode: "any" },+ },+ environment: { selectedEnvironmentId: localEnvironment.id, localEnvironment, selectedEnvironmentForConfig: localEnvironment, environmentResolution: { environmentId: localEnvironment.id, source: "default" }, resolvedInstanceSettings: await instanceSettingsService(db).get() },+ config: { mergedConfig: isolated ? { workspaceStrategy: { type: "git_worktree", baseRef: "main", branchTemplate: "test-{{issue.identifier}}" } } : {}, configSnapshot: null, secretManifest: [] },+ reuse: { requestedShouldReuseExisting: false, existingExecutionWorkspace: null, reusableExistingExecutionWorkspace: null, workspaceReuseRequest: resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: null, issueExecutionWorkspacePreference: null, existingExecutionWorkspaceStatus: null }), nativeRecoveryExecutionWorkspaceId: null, persistedNativeExecutionInput: null, isDotRun: false, runOptions: {} },+ services: { envOrchestrator, executionWorkspacesSvc: executionWorkspaceService(db), workspaceOperationsSvc: workspaceOperationService(db), issuesSvc: issueService(db), ...resolver, appendRunEvent: vi.fn() },+ controllerLease: { assertOwned: vi.fn(async () => {}) }, nativeRunnerPreparationSpans: [],+ };+ return { input, source, issue, run, envOrchestrator };+ }+ const readIssue = (id: string) => db.select().from(issues).where(eq(issues.id, id)).then(rows => rows[0]!);+ const readRun = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!);+ const leases = (id: string) => db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, id));++ async function reuse(input: HeartbeatWorkspacePreparationInput, workspace: NonNullable<Awaited<ReturnType<typeof prepareHeartbeatWorkspace>>["persistedExecutionWorkspace"]>) {+ input.reuse.requestedShouldReuseExisting = true;+ input.reuse.existingExecutionWorkspace = workspace;+ input.reuse.reusableExistingExecutionWorkspace = workspace;+ input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: workspace.id, issueExecutionWorkspacePreference: "reuse_existing", existingExecutionWorkspaceStatus: workspace.status });+ input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: workspace.id, executionWorkspacePreference: "reuse_existing" };+ const [run] = await db.insert(heartbeatRuns).values({ companyId: input.run.companyId, agentId: input.agent.id, issueId: input.task.issueId, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();+ input.run = run;+ }++ it("persists the workspace and issue binding before acquiring its environment", async () => {+ const { input, source, issue, envOrchestrator } = await fixture();+ const acquire = envOrchestrator.acquireForRun;+ vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {+ expect((await readIssue(issue.id)).executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);+ expect((await readRun(input.run.id)).contextSnapshot?.executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);+ return acquire(args);+ });+ const result = await prepareHeartbeatWorkspace(db, input);+ expect(result.executionWorkspace.cwd).toBe(source);+ expect(result.persistedExecutionWorkspace).toMatchObject({ companyId: issue.companyId, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, mode: "shared_workspace" });+ expect(await leases(input.run.id)).toMatchObject([{ status: "active", executionWorkspaceId: result.persistedExecutionWorkspace!.id }]);+ expect(input.nativeRunnerPreparationSpans.map(span => span.name)).toEqual(["environment.acquire", "environment.workspace.realize"]);+ expect(input.controllerLease.assertOwned).toHaveBeenCalledTimes(2);+ });++ it("creates a real isolated worktree and reuses its recorded workspace", async () => {+ const { input, source, issue } = await fixture(true);+ const first = await prepareHeartbeatWorkspace(db, input);+ expect(first.executionWorkspace.cwd).not.toBe(source);+ expect(await readFile(join(first.executionWorkspace.cwd, "README.md"), "utf8")).toBe("source content\n");+ expect(await readIssue(issue.id)).toMatchObject({ executionWorkspaceId: first.persistedExecutionWorkspace!.id, executionWorkspacePreference: "reuse_existing" });+ await writeFile(join(first.executionWorkspace.cwd, "retained.txt"), "keep this work");+ await reuse(input, first.persistedExecutionWorkspace!);+ const next = await prepareHeartbeatWorkspace(db, input);+ expect(next.reusedExecutionWorkspace).toMatchObject({ cwd: first.executionWorkspace.cwd });+ expect(next.persistedExecutionWorkspace!.id).toBe(first.persistedExecutionWorkspace!.id);+ expect(await readFile(join(next.executionWorkspace.cwd, "retained.txt"), "utf8")).toBe("keep this work");+ });++ it("keeps the admitted recovery workspace when the issue now points elsewhere", async () => {+ const { input, issue } = await fixture(true);+ const first = await prepareHeartbeatWorkspace(db, input);+ await reuse(input, first.persistedExecutionWorkspace!);+ await db.update(issues).set({ executionWorkspaceId: null, projectWorkspaceId: null }).where(eq(issues.id, issue.id));+ input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: null, projectWorkspaceId: null };+ input.reuse.nativeRecoveryExecutionWorkspaceId = first.persistedExecutionWorkspace!.id;+ const anchor = vi.spyOn(input.services, "resolveReusedGitWorkspaceAnchor");+ const next = await prepareHeartbeatWorkspace(db, input);+ expect(next.persistedExecutionWorkspace!.id).toBe(first.persistedExecutionWorkspace!.id);+ expect(anchor).toHaveBeenCalledWith(expect.objectContaining({ immutableNativeBinding: true, explicitProjectWorkspaceId: first.persistedExecutionWorkspace!.projectWorkspaceId }));+ expect(await readIssue(issue.id)).toMatchObject({ executionWorkspaceId: null, projectWorkspaceId: null });+ });++ it("removes a newly created worktree when persistence fails and preserves the original error", async () => {+ const { input, source } = await fixture(true);+ const error = new Error("workspace persistence unavailable");+ let createdCwd = "";+ vi.spyOn(input.services.executionWorkspacesSvc, "create").mockImplementation(async args => { createdCwd = args.cwd!; throw error; });+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error);+ expect(createdCwd).not.toBe("");+ await expect(access(createdCwd)).rejects.toThrow();+ expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");+ expect(await leases(input.run.id)).toHaveLength(0);+ });++ it("preserves a reused worktree when its metadata update fails", async () => {+ const { input } = await fixture(true);+ const first = await prepareHeartbeatWorkspace(db, input);+ await reuse(input, first.persistedExecutionWorkspace!);+ await writeFile(join(first.executionWorkspace.cwd, "retained.txt"), "unsaved work");+ const error = new Error("workspace update unavailable");+ vi.spyOn(input.services.executionWorkspacesSvc, "update").mockRejectedValueOnce(error);+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error);+ expect(await readFile(join(first.executionWorkspace.cwd, "retained.txt"), "utf8")).toBe("unsaved work");+ expect(await leases(input.run.id)).toHaveLength(0);+ });++ it.each(["acquire", "realize", "ownership"] as const)("preserves %s errors and durable lease state for outer cleanup", async failure => {+ const { input, envOrchestrator } = await fixture();+ const error = new Error(`${failure} failed`);+ if (failure === "acquire") vi.spyOn(envOrchestrator, "acquireForRun").mockRejectedValueOnce(error);+ if (failure === "realize") vi.spyOn(envOrchestrator, "realizeForRun").mockRejectedValueOnce(error);+ if (failure === "ownership") vi.mocked(input.controllerLease.assertOwned).mockResolvedValueOnce(undefined).mockRejectedValueOnce(error);+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error);+ expect(input.nativeRunnerPreparationSpans.at(-1)).toMatchObject({ name: failure === "realize" ? "environment.workspace.realize" : "environment.acquire", outcome: "failed" });+ expect((await readRun(input.run.id)).status).toBe("running");+ expect(await leases(input.run.id)).toHaveLength(failure === "acquire" ? 0 : 1);+ if (failure !== "acquire") {+ expect(await leases(input.run.id)).toMatchObject([{ status: "active" }]);+ // The production executor terminalizes first, then releases by durable run ID.+ await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, input.run.id));+ await envOrchestrator.releaseForRun({ heartbeatRunId: input.run.id, companyId: input.run.companyId, agentId: input.agent.id, status: "released" });+ expect(await leases(input.run.id)).toMatchObject([{ status: "released" }]);+ }+ });++ it("checks the acquired environment allowlist before realization", async () => {+ const { input, envOrchestrator } = await fixture();+ input.policy.executionPolicy = { managedSandboxOnly: true };+ const realize = vi.spyOn(envOrchestrator, "realizeForRun");+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("forbids local execution");+ expect(realize).not.toHaveBeenCalled();+ expect(await leases(input.run.id)).toMatchObject([{ status: "active" }]);+ });++ it("rejects low-trust local execution before resolving or provisioning a workspace", async () => {+ const { input } = await fixture(true);+ input.policy.trustPreset = {+ kind: "low_trust_review", preset: "low_trust_review", sourcePresets: {},+ boundary: { mode: "low_trust_review", companyId: input.agent.companyId, rootIssueId: input.task.issueId! },+ };+ const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");+ const create = vi.spyOn(input.services.executionWorkspacesSvc, "create");+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("requires a sandbox environment");+ expect(resolve).not.toHaveBeenCalled();+ expect(create).not.toHaveBeenCalled();+ expect(await leases(input.run.id)).toHaveLength(0);+ });++ it("rejects remote recovery without its retained sync binding before acquiring a lease", async () => {+ const { input, envOrchestrator } = await fixture();+ input.reuse.runOptions.nativeRestartRecovery = {+ kind: "reattach_remote_runner", runId: input.run.id, leaseOwner: "controller",+ controllerGeneration: 1, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,+ remote: { providerLeaseId: "original-provider-lease", remoteCwd: "/original" },+ };+ const acquire = vi.spyOn(envOrchestrator, "acquireForRun");+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("native_remote_recovery_lease_mismatch");+ expect(acquire).not.toHaveBeenCalled();+ expect(await leases(input.run.id)).toHaveLength(0);+ expect(input.nativeRunnerPreparationSpans).toMatchObject([{ name: "environment.acquire", outcome: "failed" }]);+ });++ it("binds a replacement workspace from realization and snapshots the restore lease", async () => {+ const { input, envOrchestrator, issue } = await fixture();+ const realize = envOrchestrator.realizeForRun;+ vi.spyOn(envOrchestrator, "realizeForRun").mockImplementation(async args => {+ const result = await realize(args);+ const workspace = await input.services.executionWorkspacesSvc.create({ companyId: issue.companyId, projectId: issue.projectId!, projectWorkspaceId: issue.projectWorkspaceId, sourceIssueId: issue.id, name: "Realized workspace", mode: "shared_workspace", strategyType: "project_primary", cwd: args.executionWorkspace.cwd });+ return { ...result, persistedExecutionWorkspace: workspace, lease: { ...result.lease, metadata: { remoteCwd: "/original" } } };+ });+ const result = await prepareHeartbeatWorkspace(db, input);+ expect((await readIssue(issue.id)).executionWorkspaceId).toBe(result.persistedExecutionWorkspace!.id);+ expect(result.persistedExecutionWorkspace!.name).toBe("Realized workspace");+ result.activeEnvironmentLease.lease.metadata!.remoteCwd = "/rebound";+ expect(result.workspaceRestoreSource.metadata?.remoteCwd).toBe("/original");+ });+});diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.tsnew file mode 100644index 0000000000..cb699b9482--- /dev/null+++ b/server/src/services/heartbeat/workspace-preparation.ts@@ -0,0 +1,1072 @@+import {+ resolveWorkspaceAfterLowTrustPreflight,+ WorkspaceValidationFailure,+ stripHostWorkspaceProvisionForLowTrustSandbox,+ assertGitWorktreeBaseWorkspaceReady,+ buildEffectiveRunWorkspaceConfigMetadata,+ resolveExecutionWorkspaceConfigFreshness,+ resolveExecutionWorkspaceReuseProvisioningPolicy,+ provisionExecutionWorkspaceForFreshnessDecision,+ mergeExecutionWorkspaceMetadataForPersistence,+ resolveExecutionWorkspaceBranchOwnership,+ reconcileReusedExecutionWorkspaceProjectWorkspaceId,+ recordWorkspaceConfigFreshnessOperation,+ prepareProjectRepositoryWorkspaces,+} from "./workspaces.js";+import { resolvePaperclipInstanceRoot } from "../../home-paths.js";+import fs from "node:fs/promises";+import path from "node:path";+import { and, asc, eq } from "drizzle-orm";+import { type ExecutionWorkspace } from "@paperclipai/shared";+import { heartbeatRuns, projectWorkspaces } from "@paperclipai/db";+import { getStartupTracer } from "../../instrumentation.js";+import { createHostDuplexObservabilityRecorder } from "../duplex-observability-recorder.js";+import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js";+import { logger } from "../../middleware/logger.js";+import { createGitRemoteAuthProvider } from "../git-credentials.js";+import { readNativeWorkspaceSyncReference } from "../native-runtime/index.js";+import { parseObject } from "../../adapters/utils.js";+import { materializeNativeChatTaskRoot } from "../native-runtime/native-chat-workspace.js";+import { materializeIsolatedTaskDirectory } from "../isolated-task-directory.js";+import {+ cleanupExecutionWorkspaceArtifacts,+ ensurePersistedExecutionWorkspaceAvailable,+ realizeExecutionWorkspace,+ type ExecutionWorkspaceInput,+ type RealizedExecutionWorkspace,+} from "../workspace-runtime.js";+import {+ readManagedWorktreeInstanceOwnership,+ WORKTREE_INSTANCE_ROOT_METADATA_KEY,+} from "../workspace-instance-cleanup.js";+import { isRuntimeOwnedGitBranch } from "../execution-workspace-branch-ownership.js";+import {+ issueExecutionWorkspaceModeForPersistedWorkspace,+ resolveEffectiveWorkspaceStrategyType,+} from "../execution-workspace-policy.js";+import { evaluateExecutionAllowlist } from "../execution-allowlist.js";+import type { Db, agents } from "@paperclipai/db";+import type {+ Environment,+ IssueExecutionWorkspaceSettings,+ ProjectExecutionWorkspacePolicy,+} from "@paperclipai/shared";+import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";+import type { NativeRunHistoricalSpan } from "../native-runtime/native-run-trace.js";+import type { NativeRestartRecoveryClaim } from "../native-runtime/index.js";+import type { NativeChatWorkspaceScope } from "../native-runtime/native-chat-workspace.js";+import type { TrustPresetResolution } from "../trust-preset-resolver.js";+import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js";+import type { instanceSettingsService } from "../instance-settings.js";+import type { issueService } from "../issues.js";+import type { executionWorkspaceService } from "../execution-workspaces.js";+import type { workspaceOperationService } from "../workspace-operations.js";+import type { watchLegacyControllerLease } from "../legacy-controller-lease.js";+import type {+ createHeartbeatWorkspaceResolver,+ resolveExecutionWorkspaceReuseRequestForIssue,+} from "./workspaces.js";+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";+import type {+ ParsedExecutionWorkspaceMode,+ ExecutionWorkspaceEnvironmentResolution,+} from "../execution-workspace-policy.js";++type WorkspaceResolver = ReturnType<typeof createHeartbeatWorkspaceResolver>;+type WorkspaceMetadataInput = Parameters<typeof buildEffectiveRunWorkspaceConfigMetadata>[0];++/** Prepares the workspace and environment; dispatch and terminal lease release stay with the caller. */+export interface HeartbeatWorkspacePreparationInput {+ run: typeof heartbeatRuns.$inferSelect;+ agent: typeof agents.$inferSelect;+ task: {+ issueRef: (NonNullable<Parameters<typeof realizeExecutionWorkspace>[0]["issue"]> & {+ projectId: string | null;+ projectWorkspaceId: string | null;+ executionWorkspaceId: string | null;+ executionWorkspacePreference: string | null;+ }) | null;+ issueId: string | null;+ context: Record<string, unknown>;+ executionProjectId: string | null;+ responsibleUserId: string | null;+ previousSessionParams: Record<string, unknown> | null;+ };+ policy: {+ trustPreset: TrustPresetResolution;+ isolatedWorkspacesEnabled: boolean;+ effectiveExecutionWorkspaceMode: ParsedExecutionWorkspaceMode;+ requestedExecutionWorkspaceMode: ParsedExecutionWorkspaceMode;+ useIsolatedTaskDirectory: boolean;+ nativeChatWorkspaceScope: NativeChatWorkspaceScope | null;+ projectExecutionWorkspacePolicy: ProjectExecutionWorkspacePolicy | null;+ issueExecutionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;+ environmentExecutionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;+ executionPolicy: Parameters<typeof evaluateExecutionAllowlist>[0];+ };+ environment: {+ selectedEnvironmentId: string;+ localEnvironment: Environment;+ selectedEnvironmentForConfig: Environment | null;+ environmentResolution: ExecutionWorkspaceEnvironmentResolution;+ resolvedInstanceSettings: Awaited<ReturnType<ReturnType<typeof instanceSettingsService>["get"]>>;+ };+ config: {+ mergedConfig: Record<string, unknown>;+ configSnapshot: WorkspaceMetadataInput["configSnapshot"];+ secretManifest: WorkspaceMetadataInput["secretManifest"];+ };+ reuse: {+ requestedShouldReuseExisting: boolean;+ existingExecutionWorkspace: ExecutionWorkspace | null;+ reusableExistingExecutionWorkspace: ExecutionWorkspace | null;+ workspaceReuseRequest: ReturnType<typeof resolveExecutionWorkspaceReuseRequestForIssue>;+ nativeRecoveryExecutionWorkspaceId: string | null;+ persistedNativeExecutionInput: NativeExecutionInput | null;+ isDotRun: boolean;+ runOptions: { nativeRestartRecovery?: NativeRestartRecoveryClaim };+ };+ services: {+ envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "resolveEnvironment" | "acquireForRun" | "realizeForRun">;+ executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "create" | "update">;+ workspaceOperationsSvc: Pick<ReturnType<typeof workspaceOperationService>, "createRecorder">;+ issuesSvc: Pick<ReturnType<typeof issueService>, "update">;+ resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"];+ resolveReusedGitWorkspaceAnchor: WorkspaceResolver["resolveReusedGitWorkspaceAnchor"];+ appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];+ };+ controllerLease: Pick<ReturnType<typeof watchLegacyControllerLease>, "assertOwned">;+ nativeRunnerPreparationSpans: NativeRunHistoricalSpan[];+}++function readNonEmptyString(value: unknown): string | null {+ return typeof value === "string" && value.trim().length > 0 ? value : null;+}++export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) {+ const { run, agent, controllerLease, nativeRunnerPreparationSpans } = input;+ const {+ issueRef,+ issueId,+ context,+ executionProjectId,+ responsibleUserId,+ previousSessionParams,+ } = input.task;+ const {+ trustPreset,+ isolatedWorkspacesEnabled,+ effectiveExecutionWorkspaceMode,+ requestedExecutionWorkspaceMode,+ useIsolatedTaskDirectory,+ nativeChatWorkspaceScope,+ projectExecutionWorkspacePolicy,+ issueExecutionWorkspaceSettings,+ environmentExecutionWorkspaceSettings,+ executionPolicy,+ } = input.policy;+ const {+ selectedEnvironmentId,+ localEnvironment,+ selectedEnvironmentForConfig,+ environmentResolution,+ resolvedInstanceSettings,+ } = input.environment;+ const { mergedConfig, configSnapshot, secretManifest } = input.config;+ const {+ requestedShouldReuseExisting,+ existingExecutionWorkspace,+ reusableExistingExecutionWorkspace,+ workspaceReuseRequest,+ nativeRecoveryExecutionWorkspaceId,+ persistedNativeExecutionInput,+ isDotRun,+ runOptions,+ } = input.reuse;+ const {+ envOrchestrator,+ executionWorkspacesSvc,+ workspaceOperationsSvc,+ issuesSvc,+ resolveWorkspaceForRun,+ resolveReusedGitWorkspaceAnchor,+ appendRunEvent,+ } = input.services;+ const {+ selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,+ workspace: resolvedWorkspace,+ } = await resolveWorkspaceAfterLowTrustPreflight({+ db,+ trustPreset,+ isolatedWorkspacesEnabled,+ effectiveExecutionWorkspaceMode,+ issue: issueRef+ ? {+ companyId: agent.companyId,+ id: issueRef.id,+ projectId: issueRef.projectId,+ }+ : null,+ resolveSelectedEnvironmentDriver: async () => {+ const preflightEnvironment = await envOrchestrator.resolveEnvironment(+ {+ companyId: agent.companyId,+ selectedEnvironmentId,+ localEnvironmentId: localEnvironment.id,+ },+ );+ return preflightEnvironment.driver;+ },+ resolveWorkspace: async () => {+ if (isDotRun) {+ // This is private controller storage, never a provider filesystem.+ // v6 projects workspace.access=none and cwd=null to Dot.+ const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "dot-controllers", agent.companyId, run.id);+ await fs.mkdir(cwd, { recursive: true, mode: 0o700 });+ return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null,+ workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] };+ }+ if (useIsolatedTaskDirectory && issueRef) {+ const cwd = await materializeIsolatedTaskDirectory({+ companyId: agent.companyId,+ issueId: issueRef.id,+ });+ if (reusableExistingExecutionWorkspace && (+ reusableExistingExecutionWorkspace.companyId !== agent.companyId ||+ reusableExistingExecutionWorkspace.projectId !== issueRef.projectId ||+ reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id ||+ reusableExistingExecutionWorkspace.mode !== "isolated_workspace" ||+ reusableExistingExecutionWorkspace.strategyType !== "project_primary" ||+ reusableExistingExecutionWorkspace.cwd !== cwd+ )) {+ throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", {+ workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id },+ });+ }+ return resolveWorkspaceForRun(agent, context, previousSessionParams, {+ executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null,+ anchorWorkspace: {+ cwd,+ source: "task_session",+ projectId: issueRef.projectId,+ workspaceId: null,+ repoUrl: null,+ repoRef: null,+ workspaceHints: [],+ warnings: [],+ baseCwdFallback: false,+ materializationFailures: [],+ },+ });+ }+ if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) {+ const cwd = await materializeNativeChatTaskRoot(+ nativeChatWorkspaceScope,+ );+ return {+ cwd,+ source: "task_session" as const,+ projectId: null,+ workspaceId: null,+ repoUrl: null,+ repoRef: null,+ workspaceHints: [],+ warnings: [],+ baseCwdFallback: false,+ materializationFailures: [],+ additionalWorkspaces: [],+ referencedProjectFailures: [],+ };+ }+ const workspace = await resolveWorkspaceForRun(+ agent,+ context,+ previousSessionParams,+ {+ useProjectWorkspace:+ requestedExecutionWorkspaceMode !== "agent_default",+ anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"+ ? await resolveReusedGitWorkspaceAnchor({+ agent,+ workspace: reusableExistingExecutionWorkspace,+ responsibleUserId,+ immutableNativeBinding: Boolean(nativeRecoveryExecutionWorkspaceId),+ projectId: nativeRecoveryExecutionWorkspaceId+ ? reusableExistingExecutionWorkspace.projectId+ : issueRef?.projectId ?? readNonEmptyString(context.projectId),+ explicitProjectWorkspaceId: nativeRecoveryExecutionWorkspaceId+ ? reusableExistingExecutionWorkspace.projectWorkspaceId+ : readNonEmptyString(context.projectWorkspaceId),+ issueId,+ runId: run.id,+ })+ : undefined,+ // Thread the selected environment driver so run-workspace resolution can tell a local+ // target from a remote one, and a confined sandbox target from an unconfined remote+ // target. A remote run resolves referenced projects only for the confined sandbox+ // transport with the remote flag on. This never changes the anchor workspace.+ executionEnvironmentDriver:+ selectedEnvironmentForConfig?.driver ?? null,+ },+ );+ // Additional referenced projects are a separate trusted Board+ // capability, not extra readable roots for an external conversation.+ return nativeChatWorkspaceScope+ ? {+ ...workspace,+ additionalWorkspaces: [],+ referencedProjectFailures: [],+ }+ : workspace;+ },+ });+ const hostExecutionWorkspaceConfig = isDotRun ? {} :+ stripHostWorkspaceProvisionForLowTrustSandbox({+ config: mergedConfig,+ trustPreset,+ selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,+ });+ const executionWorkspaceBase = {+ baseCwd: resolvedWorkspace.cwd,+ source: resolvedWorkspace.source,+ projectId: resolvedWorkspace.projectId,+ workspaceId: resolvedWorkspace.workspaceId,+ repoUrl: resolvedWorkspace.repoUrl,+ repoRef: resolvedWorkspace.repoRef,+ additionalWorkspaces: resolvedWorkspace.additionalWorkspaces,+ } satisfies ExecutionWorkspaceInput;+ await assertGitWorktreeBaseWorkspaceReady({+ requestedExecutionWorkspaceMode,+ config: hostExecutionWorkspaceConfig,+ issue: issueRef,+ base: executionWorkspaceBase,+ anchor: {+ baseCwdFallback: resolvedWorkspace.baseCwdFallback,+ materializationFailures: resolvedWorkspace.materializationFailures,+ localPathOnlyWorkspace: resolvedWorkspace.localPathOnlyWorkspace,+ },+ });+ const workspaceStrategyForFingerprint = parseObject(+ hostExecutionWorkspaceConfig.workspaceStrategy,+ );+ const workspaceStrategyFingerprintValue =+ Object.keys(workspaceStrategyForFingerprint).length > 0+ ? workspaceStrategyForFingerprint+ : null;+ const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType(+ requestedExecutionWorkspaceMode,+ hostExecutionWorkspaceConfig,+ );+ const selectedEnvironmentConfigForFingerprint = parseObject(+ selectedEnvironmentForConfig?.config,+ );+ const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig+ ? {+ selectionSource: environmentResolution.source,+ selectedEnvironmentId,+ driver: selectedEnvironmentForConfig.driver,+ provider: readNonEmptyString(+ selectedEnvironmentConfigForFingerprint.provider,+ ),+ config: selectedEnvironmentForConfig.config,+ configRevisionAt:+ selectedEnvironmentForConfig.updatedAt instanceof Date+ ? selectedEnvironmentForConfig.updatedAt.toISOString()+ : (selectedEnvironmentForConfig.updatedAt ?? null),+ executionPolicy,+ }+ : null;+ const workspaceRealizationFingerprint = {+ environmentDriver: selectedEnvironmentForConfig?.driver ?? null,+ environmentProvider: readNonEmptyString(+ selectedEnvironmentConfigForFingerprint.provider,+ ),+ trustPreset: trustPreset.kind,+ lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver,+ };+ const workspaceFreshnessSource = resolvedWorkspace.freshnessSource ?? executionWorkspaceBase;+ const latestWorkspaceConfigMetadata =+ buildEffectiveRunWorkspaceConfigMetadata({+ mode: requestedExecutionWorkspaceMode,+ projectId: workspaceFreshnessSource.projectId,+ projectWorkspaceId: workspaceFreshnessSource.workspaceId,+ strategyType: latestWorkspaceStrategyType,+ workspaceStrategy: workspaceStrategyFingerprintValue,+ repoUrl: workspaceFreshnessSource.repoUrl,+ repoRef:+ readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ??+ workspaceFreshnessSource.repoRef,+ configSnapshot,+ environment: workspaceEnvironmentFingerprint,+ realization: workspaceRealizationFingerprint,+ secretManifest,+ });+ const inferredExistingWorkspaceConfigMetadata =+ reusableExistingExecutionWorkspace+ ? buildEffectiveRunWorkspaceConfigMetadata({+ mode: issueExecutionWorkspaceModeForPersistedWorkspace(+ reusableExistingExecutionWorkspace.mode,+ ),+ projectId: reusableExistingExecutionWorkspace.projectId,+ projectWorkspaceId:+ reusableExistingExecutionWorkspace.projectWorkspaceId,+ strategyType: reusableExistingExecutionWorkspace.strategyType,+ workspaceStrategy: workspaceStrategyFingerprintValue+ ? {+ ...workspaceStrategyFingerprintValue,+ type: reusableExistingExecutionWorkspace.strategyType,+ ...(reusableExistingExecutionWorkspace.baseRef+ ? { baseRef: reusableExistingExecutionWorkspace.baseRef }+ : {}),+ }+ : { type: reusableExistingExecutionWorkspace.strategyType },+ repoUrl: reusableExistingExecutionWorkspace.repoUrl,+ repoRef: reusableExistingExecutionWorkspace.baseRef,+ configSnapshot: reusableExistingExecutionWorkspace.config,+ environment: workspaceEnvironmentFingerprint,+ realization: workspaceRealizationFingerprint,+ secretManifest,+ evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt,+ })+ : null;+ const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness(+ {+ hasExistingWorkspace:+ requestedShouldReuseExisting &&+ Boolean(reusableExistingExecutionWorkspace),+ existingWorkspaceMetadata:+ reusableExistingExecutionWorkspace?.metadata ?? null,+ inferredMetadata: inferredExistingWorkspaceConfigMetadata,+ nextMetadata: latestWorkspaceConfigMetadata,+ },+ );+ const workspaceReuseProvisioningPolicy =+ resolveExecutionWorkspaceReuseProvisioningPolicy({+ requestedShouldReuseExisting,+ workspaceConfigFreshness,+ });+ const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({+ companyId: agent.companyId,+ heartbeatRunId: run.id,+ executionWorkspaceId:+ workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace+ ? workspaceReuseRequest.requestedExecutionWorkspaceId+ : null,+ issueId,+ });+ // The run-scoped provider resolves the active identity at each Git operation,+ // including base-ref refreshes, workspace realization, and restore.+ const workspaceGitAuthProvider = createGitRemoteAuthProvider(+ db,+ agent.companyId,+ {+ issueId,+ heartbeatRunId: run.id,+ responsibleUserId: run.responsibleUserId,+ agentId: agent.id,+ },+ );+ const {+ executionWorkspace,+ reusedExecutionWorkspace,+ policy: resolvedWorkspaceReusePolicy,+ } = isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(+ {+ requestedShouldReuseExisting,+ existingExecutionWorkspaceId:+ workspaceReuseRequest.requestedExecutionWorkspaceId,+ issueRef,+ runId: run.id,+ workspaceConfigFreshness,+ restoreExistingWorkspace: reusableExistingExecutionWorkspace+ ? () =>+ ensurePersistedExecutionWorkspaceAvailable({+ db,+ base: executionWorkspaceBase,+ workspace: {+ id: reusableExistingExecutionWorkspace.id,+ mode: reusableExistingExecutionWorkspace.mode,+ strategyType:+ reusableExistingExecutionWorkspace.strategyType,+ cwd: reusableExistingExecutionWorkspace.cwd,+ providerRef: reusableExistingExecutionWorkspace.providerRef,+ projectId: reusableExistingExecutionWorkspace.projectId,+ projectWorkspaceId:+ reusableExistingExecutionWorkspace.projectWorkspaceId,+ repoUrl: reusableExistingExecutionWorkspace.repoUrl,+ baseRef: reusableExistingExecutionWorkspace.baseRef,+ branchName: reusableExistingExecutionWorkspace.branchName,+ metadata:+ reusableExistingExecutionWorkspace.metadata as Record<+ string,+ unknown+ > | null,+ config: {+ provisionCommand:+ configSnapshot?.provisionCommand ??+ reusableExistingExecutionWorkspace.config+ ?.provisionCommand ??+ projectExecutionWorkspacePolicy?.workspaceStrategy+ ?.provisionCommand ??+ null,+ runtimeProvisionCommand:+ configSnapshot?.runtimeProvisionCommand ??+ reusableExistingExecutionWorkspace.config+ ?.runtimeProvisionCommand ??+ projectExecutionWorkspacePolicy?.workspaceStrategy+ ?.runtimeProvisionCommand ??+ null,+ },+ },+ issue: issueRef,+ agent: {+ id: agent.id,+ name: agent.name,+ companyId: agent.companyId,+ },+ heartbeatRunId: run.id,+ enableWorkspaceBranchReconcileForward:+ resolvedInstanceSettings.experimental+ .enableWorkspaceBranchReconcileForward,+ enableWorkspaceDirtyQuarantineRepair:+ resolvedInstanceSettings.experimental+ .enableWorkspaceDirtyQuarantineRepair,+ recorder: workspaceOperationRecorder,+ resolveGitAuth: workspaceGitAuthProvider,+ })+ : null,+ realizeWorkspace: () =>+ realizeExecutionWorkspace({+ db,+ base: executionWorkspaceBase,+ config: hostExecutionWorkspaceConfig,+ issue: issueRef,+ agent: {+ id: agent.id,+ name: agent.name,+ companyId: agent.companyId,+ },+ recordedBranchOwnership:+ existingExecutionWorkspace?.status !== "archived" &&+ existingExecutionWorkspace?.branchName+ ? {+ branchName: existingExecutionWorkspace.branchName,+ createdByRuntime: isRuntimeOwnedGitBranch(+ existingExecutionWorkspace.metadata,+ ),+ }+ : null,+ heartbeatRunId: run.id,+ enableWorkspaceBranchReconcileForward:+ resolvedInstanceSettings.experimental+ .enableWorkspaceBranchReconcileForward,+ enableWorkspaceDirtyQuarantineRepair:+ resolvedInstanceSettings.experimental+ .enableWorkspaceDirtyQuarantineRepair,+ recorder: workspaceOperationRecorder,+ resolveGitAuth: workspaceGitAuthProvider,+ }),+ },+ );+ const resolvedProjectId =+ executionWorkspace.projectId ??+ issueRef?.projectId ??+ executionProjectId ??+ null;+ const resolvedProjectWorkspaceId =+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"+ ? reusableExistingExecutionWorkspace.projectWorkspaceId+ : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;+ let persistedExecutionWorkspace: ExecutionWorkspace | null = null;+ let issueExecutionWorkspaceIdForRun =+ issueRef?.executionWorkspaceId ?? null;+ let issueProjectWorkspaceIdForRun = issueRef?.projectWorkspaceId ?? null;+ let issueExecutionWorkspacePreferenceForRun =+ issueRef?.executionWorkspacePreference ?? null;+ let issueExecutionWorkspaceModeForRun =+ issueExecutionWorkspaceSettings?.mode ?? null;+ const warmReusableExecutionWorkspace =+ selectedEnvironmentForConfig?.driver === "sandbox" &&+ selectedEnvironmentConfigForFingerprint.reuseLease === true &&+ selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm";+ // Native provider checkpoints bind to the workspace row, including ordinary+ // local shared workspaces. Persist that binding independently of the opt-in+ // isolated-workspace UI, just as warm sandbox continuity already does.+ const nativeSharedWorkspace = agent.adapterType === "paperclip_runner" &&+ requestedExecutionWorkspaceMode === "shared_workspace";+ const bindIssueToPersistedExecutionWorkspace = async (+ workspace: ExecutionWorkspace | null,+ ) => {+ if (!issueId || !workspace || nativeRecoveryExecutionWorkspaceId) {+ return;+ }+ const nextIssueWorkspaceMode =+ issueExecutionWorkspaceModeForPersistedWorkspace(workspace.mode) ??+ "agent_default";+ const shouldSwitchIssueToExistingWorkspace =+ issueRef?.executionWorkspacePreference === "reuse_existing" ||+ requestedExecutionWorkspaceMode === "isolated_workspace" ||+ requestedExecutionWorkspaceMode === "operator_branch" ||+ warmReusableExecutionWorkspace || nativeSharedWorkspace;+ const nextIssuePatch: Record<string, unknown> = {};+ if (issueExecutionWorkspaceIdForRun !== workspace.id) {+ nextIssuePatch.executionWorkspaceId = workspace.id;+ }+ if (+ resolvedProjectWorkspaceId &&+ issueProjectWorkspaceIdForRun !== resolvedProjectWorkspaceId+ ) {+ nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;+ }+ if (+ shouldSwitchIssueToExistingWorkspace &&+ (issueExecutionWorkspacePreferenceForRun !== "reuse_existing" ||+ issueExecutionWorkspaceModeForRun !== nextIssueWorkspaceMode)+ ) {+ nextIssuePatch.executionWorkspacePreference = "reuse_existing";+ nextIssuePatch.executionWorkspaceSettings = {+ ...(issueExecutionWorkspaceSettings ?? {}),+ mode: nextIssueWorkspaceMode,+ };+ }+ if (Object.keys(nextIssuePatch).length > 0) {+ await issuesSvc.update(+ issueId,+ { ...nextIssuePatch, companyGuard: agent.companyId },+ db,+ undefined,+ undefined,+ { bindRuntimeSharedWorkspace: (warmReusableExecutionWorkspace || nativeSharedWorkspace) && workspace.mode === "shared_workspace" },+ );+ issueExecutionWorkspaceIdForRun = workspace.id;+ issueProjectWorkspaceIdForRun =+ resolvedProjectWorkspaceId ?? issueProjectWorkspaceIdForRun;+ if (shouldSwitchIssueToExistingWorkspace) {+ issueExecutionWorkspacePreferenceForRun = "reuse_existing";+ issueExecutionWorkspaceModeForRun = nextIssueWorkspaceMode;+ }+ }+ };+ const baseExecutionWorkspaceMetadata =+ mergeExecutionWorkspaceMetadataForPersistence({+ existingMetadata:+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace+ ? (reusableExistingExecutionWorkspace?.metadata ?? null)+ : null,+ source: executionWorkspace.source,+ // Attaching a new worktree to a pre-existing branch reports a fresh+ // workspace, but must not make cleanup own the operator's branch.+ createdByRuntime:+ resolveExecutionWorkspaceBranchOwnership(executionWorkspace),+ strategyType: executionWorkspace.strategy,+ configSnapshot,+ shouldReuseExisting:+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace,+ shouldRefreshConfigSnapshot:+ resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,+ workspaceConfigMetadata:+ resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata+ ? latestWorkspaceConfigMetadata+ : null,+ baseRef: executionWorkspace.repoRef,+ baseRefSha: executionWorkspace.baseRefSha ?? null,+ });+ let persistedWorktreeInstanceRoot =+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&+ typeof reusableExistingExecutionWorkspace?.metadata?.[+ WORKTREE_INSTANCE_ROOT_METADATA_KEY+ ] === "string"+ ? reusableExistingExecutionWorkspace.metadata[+ WORKTREE_INSTANCE_ROOT_METADATA_KEY+ ]+ : null;+ if (+ !persistedWorktreeInstanceRoot &&+ executionWorkspace.strategy === "git_worktree" &&+ executionWorkspace.worktreePath+ ) {+ try {+ persistedWorktreeInstanceRoot =+ (+ await readManagedWorktreeInstanceOwnership(+ executionWorkspace.worktreePath,+ )+ )?.instanceRoot ?? null;+ } catch (error) {+ logger.warn(+ {+ runId: run.id,+ issueId,+ executionWorkspaceCwd: executionWorkspace.cwd,+ error: error instanceof Error ? error.message : String(error),+ },+ "Could not record managed worktree instance ownership",+ );+ }+ }+ const nextExecutionWorkspaceMetadata = {+ ...baseExecutionWorkspaceMetadata,+ ...(persistedWorktreeInstanceRoot+ ? {+ [WORKTREE_INSTANCE_ROOT_METADATA_KEY]:+ persistedWorktreeInstanceRoot,+ }+ : {}),+ };+ const pendingForwardBranchReconcile =+ executionWorkspace.pendingForwardBranchReconcile ?? null;+ const branchNameForInitialPersistence =+ pendingForwardBranchReconcile?.recordedBranchName ??+ executionWorkspace.branchName;+ try {+ persistedExecutionWorkspace =+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&+ reusableExistingExecutionWorkspace+ ? await executionWorkspacesSvc.update(+ reusableExistingExecutionWorkspace.id,+ {+ cwd: executionWorkspace.cwd,+ repoUrl: executionWorkspace.repoUrl,+ baseRef: executionWorkspace.repoRef,+ branchName: branchNameForInitialPersistence,+ providerType:+ executionWorkspace.strategy === "git_worktree"+ ? "git_worktree"+ : "local_fs",+ providerRef: executionWorkspace.worktreePath,+ status: "active",+ lastUsedAt: new Date(),+ metadata: nextExecutionWorkspaceMetadata,+ projectWorkspaceId:+ reconcileReusedExecutionWorkspaceProjectWorkspaceId(+ reusableExistingExecutionWorkspace.projectWorkspaceId,+ resolvedProjectWorkspaceId,+ ),+ },+ )+ : resolvedProjectId+ ? await executionWorkspacesSvc.create({+ companyId: agent.companyId,+ projectId: resolvedProjectId,+ projectWorkspaceId: resolvedProjectWorkspaceId,+ sourceIssueId: issueRef?.id ?? null,+ mode:+ requestedExecutionWorkspaceMode === "isolated_workspace"+ ? "isolated_workspace"+ : requestedExecutionWorkspaceMode === "operator_branch"+ ? "operator_branch"+ : requestedExecutionWorkspaceMode === "agent_default"+ ? "adapter_managed"+ : "shared_workspace",+ strategyType:+ executionWorkspace.strategy === "git_worktree"+ ? "git_worktree"+ : "project_primary",+ name:+ branchNameForInitialPersistence ??+ issueRef?.identifier ??+ `workspace-${agent.id.slice(0, 8)}`,+ status: "active",+ cwd: executionWorkspace.cwd,+ repoUrl: executionWorkspace.repoUrl,+ baseRef: executionWorkspace.repoRef,+ branchName: branchNameForInitialPersistence,+ providerType:+ executionWorkspace.strategy === "git_worktree"+ ? "git_worktree"+ : "local_fs",+ providerRef: executionWorkspace.worktreePath,+ lastUsedAt: new Date(),+ openedAt: new Date(),+ metadata: nextExecutionWorkspaceMetadata,+ })+ : null;+ } catch (error) {+ if (executionWorkspace.created) {+ try {+ await cleanupExecutionWorkspaceArtifacts({+ workspace: {+ id:+ reusableExistingExecutionWorkspace?.id ??+ workspaceReuseRequest.requestedExecutionWorkspaceId ??+ `transient-${run.id}`,+ cwd: executionWorkspace.cwd,+ providerType:+ executionWorkspace.strategy === "git_worktree"+ ? "git_worktree"+ : "local_fs",+ providerRef: executionWorkspace.worktreePath,+ branchName: executionWorkspace.branchName,+ repoUrl: executionWorkspace.repoUrl,+ baseRef: executionWorkspace.repoRef,+ projectId: resolvedProjectId,+ projectWorkspaceId: resolvedProjectWorkspaceId,+ sourceIssueId: issueRef?.id ?? null,+ metadata: nextExecutionWorkspaceMetadata,+ },+ projectWorkspace: {+ cwd: resolvedWorkspace.cwd,+ cleanupCommand: null,+ },+ cleanupCommand: configSnapshot?.cleanupCommand ?? null,+ teardownCommand:+ configSnapshot?.teardownCommand ??+ projectExecutionWorkspacePolicy?.workspaceStrategy+ ?.teardownCommand ??+ null,+ recorder: workspaceOperationRecorder,+ });+ } catch (cleanupError) {+ logger.warn(+ {+ runId: run.id,+ issueId,+ executionWorkspaceCwd: executionWorkspace.cwd,+ cleanupError:+ cleanupError instanceof Error+ ? cleanupError.message+ : String(cleanupError),+ },+ "Failed to cleanup realized execution workspace after persistence failure",+ );+ }+ }+ throw error;+ }+ await workspaceOperationRecorder.attachExecutionWorkspaceId(+ persistedExecutionWorkspace?.id ?? null,+ );+ await recordWorkspaceConfigFreshnessOperation({+ recorder: workspaceOperationRecorder,+ runId: run.id,+ decision: workspaceConfigFreshness,+ hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace),+ reuseRequested: requestedShouldReuseExisting,+ workspaceReused: Boolean(reusedExecutionWorkspace),+ configSnapshotRefreshed:+ resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,+ previousWorkspaceId:+ workspaceReuseRequest.requestedExecutionWorkspaceId,+ activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,+ });+ if (+ reusableExistingExecutionWorkspace &&+ persistedExecutionWorkspace &&+ reusableExistingExecutionWorkspace.id !==+ persistedExecutionWorkspace.id &&+ reusableExistingExecutionWorkspace.status === "active"+ ) {+ await executionWorkspacesSvc.update(+ reusableExistingExecutionWorkspace.id,+ {+ status: "idle",+ cleanupReason: null,+ },+ );+ }+ await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);+ const projectRepositoryPaths: string[] = [];+ if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) {+ const repositoryRows = await db.select().from(projectWorkspaces).where(and(+ eq(projectWorkspaces.companyId, agent.companyId),+ eq(projectWorkspaces.projectId, executionWorkspace.projectId),+ )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id));+ const repositories = await prepareProjectRepositoryWorkspaces({+ cwd: executionWorkspace.cwd,+ anchorRepoUrl: executionWorkspace.repoUrl,+ workspaces: repositoryRows,+ resolveGitAuth: workspaceGitAuthProvider,+ });+ const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd]));+ projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd)));+ if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd);+ resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({+ ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd,+ }));+ }+ if (persistedExecutionWorkspace) {+ context.executionWorkspaceId = persistedExecutionWorkspace.id;+ await db+ .update(heartbeatRuns)+ .set({+ contextSnapshot: context,+ updatedAt: new Date(),+ })+ .where(eq(heartbeatRuns.id, run.id));+ }+ const environmentAcquireStartedAtMs = Date.now();+ let acquiredEnvironment: Awaited<+ ReturnType<typeof envOrchestrator.acquireForRun>+ >;+ try {+ await controllerLease.assertOwned();+ const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner"+ ? runOptions.nativeRestartRecovery : null;+ const recoveryWorkspace = remoteRecovery+ ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null;+ if (remoteRecovery && (!recoveryWorkspace || remoteRecovery.runId !== run.id ||+ recoveryWorkspace.providerLeaseId !== remoteRecovery.remote.providerLeaseId ||+ recoveryWorkspace.remoteCwd !== remoteRecovery.remote.remoteCwd)) {+ throw new Error("native_remote_recovery_lease_mismatch");+ }+ acquiredEnvironment = await envOrchestrator.acquireForRun({+ companyId: agent.companyId,+ selectedEnvironmentId,+ localEnvironmentId: localEnvironment.id,+ adapterType: agent.adapterType,+ adapterConfig: parseObject(agent.adapterConfig),+ admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode,+ issueId: issueId ?? null,+ heartbeatRunId: run.id,+ agentId: agent.id,+ persistedExecutionWorkspace,+ executionWorkspaceSettings: environmentExecutionWorkspaceSettings,+ ...(remoteRecovery && recoveryWorkspace ? { reattachRemoteLease: {+ leaseId: recoveryWorkspace.leaseId,+ providerLeaseId: remoteRecovery.remote.providerLeaseId,+ remoteCwd: remoteRecovery.remote.remoteCwd,+ } } : {}),+ });+ await controllerLease.assertOwned();+ nativeRunnerPreparationSpans.push({+ name: "environment.acquire",+ parentName: "task.run",+ startedAtMs: environmentAcquireStartedAtMs,+ endedAtMs: Date.now(),+ attributes: { adapter: agent.adapterType },+ });+ } catch (error) {+ nativeRunnerPreparationSpans.push({+ name: "environment.acquire",+ parentName: "task.run",+ startedAtMs: environmentAcquireStartedAtMs,+ endedAtMs: Date.now(),+ outcome: "failed",+ attributes: { adapter: agent.adapterType },+ });+ throw error;+ }+ const selectedEnvironment = acquiredEnvironment.environment;+ // Defense-in-depth: re-check the actually-acquired environment against the+ // execution allowlist. Even if selection were bypassed, a denied (local/ssh/+ // non-k8s) environment FAILS the run here rather than executing untrusted.+ const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, {+ driver: selectedEnvironment.driver,+ provider:+ typeof selectedEnvironment.config?.provider === "string"+ ? selectedEnvironment.config.provider+ : null,+ });+ if (!allowlistDecision.allowed) {+ logger.error(+ {+ runId: run.id,+ issueId,+ agentId: agent.id,+ environmentId: selectedEnvironment.id,+ deniedDriver: allowlistDecision.deniedDriver,+ deniedProvider: allowlistDecision.deniedProvider,+ },+ "Execution allowlist denied the resolved environment; failing run",+ );+ throw new Error(allowlistDecision.reason);+ }+ let activeEnvironmentLease = {+ environment: acquiredEnvironment.environment,+ lease: acquiredEnvironment.lease,+ leaseContext: acquiredEnvironment.leaseContext,+ };+ const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder(+ {+ tracer: getStartupTracer(),+ incrementCounter: (metric) => {+ void incrementToolRuntimeMetricCounter(db, {+ companyId: run.companyId,+ metric,+ }).catch(() => {});+ },+ emitTransportEvent: (event) => {+ void (async () => {+ await appendRunEvent(run, {+ eventType: event.name,+ stream: "system",+ level: event.dimensions.outcome === "error" ? "warn" : "info",+ payload: { ...event.dimensions },+ });+ })().catch(() => {});+ },+ },+ );+ const environmentRealizeStartedAtMs = Date.now();+ let realizationResult: Awaited<+ ReturnType<typeof envOrchestrator.realizeForRun>+ >;+ try {+ realizationResult = await envOrchestrator.realizeForRun({+ environment: selectedEnvironment,+ lease: activeEnvironmentLease.lease,+ adapterType: agent.adapterType,+ companyId: agent.companyId,+ issueId: issueId ?? null,+ heartbeatRunId: run.id,+ executionWorkspace,+ effectiveExecutionWorkspaceMode,+ persistedExecutionWorkspace,+ duplexObservabilityRecorder,+ });+ nativeRunnerPreparationSpans.push({+ name: "environment.workspace.realize",+ parentName: "task.run",+ startedAtMs: environmentRealizeStartedAtMs,+ endedAtMs: Date.now(),+ attributes: { driver: selectedEnvironment.driver },+ });+ } catch (error) {+ nativeRunnerPreparationSpans.push({+ name: "environment.workspace.realize",+ parentName: "task.run",+ startedAtMs: environmentRealizeStartedAtMs,+ endedAtMs: Date.now(),+ outcome: "failed",+ attributes: { driver: selectedEnvironment.driver },+ });+ throw error;+ }+ const environmentRealizeEndedAtMs = Date.now();+ activeEnvironmentLease = {+ ...activeEnvironmentLease,+ lease: realizationResult.lease,+ };+ persistedExecutionWorkspace =+ realizationResult.persistedExecutionWorkspace;+ // A sandbox realization may materialize or replace the durable workspace+ // after the host-side provisioning boundary above. Bind that final ID to+ // the issue before dispatch so warm turns reuse the exact same workspace+ // and lease scope instead of silently creating a per-run replacement.+ await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);+ const workspaceRealization = realizationResult.workspaceRealization;+ const executionTarget = realizationResult.executionTarget;+ // Preserve the host-owned source before adapter context can share lease+ // metadata. A later copy-back failure must not adopt a rebound source.+ const workspaceRestoreSource = structuredClone(realizationResult.lease);+ return {+ resolvedWorkspace,+ hostExecutionWorkspaceConfig,+ latestWorkspaceConfigMetadata,+ workspaceConfigFreshness,+ workspaceOperationRecorder,+ executionWorkspace,+ reusedExecutionWorkspace,+ resolvedWorkspaceReusePolicy,+ persistedExecutionWorkspace,+ projectRepositoryPaths,+ environmentAcquireStartedAtMs,+ selectedEnvironment,+ activeEnvironmentLease,+ realizationResult,+ environmentRealizeEndedAtMs,+ workspaceRealization,+ executionTarget,+ workspaceRestoreSource,+ };+}diff --git a/tests/e2e/composer-stop.spec.ts b/tests/e2e/composer-stop.spec.tsindex e5030d6f7b..46cc882fe1 100644--- a/tests/e2e/composer-stop.spec.ts+++ b/tests/e2e/composer-stop.spec.ts@@ -205,6 +205,9 @@ for (const adapter of ["process", "paperclip_runner"] as const) {"-e","console.log('stop fixture ready'); setInterval(() => console.log('working'), 200);",],+ // The child only runs this script; server-only fixture+ // loaders must not follow NODE_OPTIONS into the adapter.+ env: { NODE_OPTIONS: "" },graceSec: 1,}: { provider: "codex", model: "gpt-5.1-codex-mini" },diff --git a/tests/e2e/workspace-bootstrap/README.md b/tests/e2e/workspace-bootstrap/README.mdindex 29f9458a8b..3bf25b641e 100644--- a/tests/e2e/workspace-bootstrap/README.md+++ b/tests/e2e/workspace-bootstrap/README.md@@ -9,14 +9,13 @@ timeout only when the test explicitly arms a marker inside its repository.From a development worktree, start the foreground instance:```sh-BOOTSTRAP_FIXTURE_KEY=not-a-provider-key \-PATH="$PWD/tests/e2e/workspace-bootstrap/bin:$PATH" \-NODE_ENV=test \-node cli/node_modules/tsx/dist/cli.mjs cli/src/index.ts test-drive \- --harness codex --api-key-env BOOTSTRAP_FIXTURE_KEY \- --company-name 'Workspace Recovery QA' --no-browser+node cli/node_modules/tsx/dist/cli.mjs tests/e2e/workspace-bootstrap/launch.ts```+The launcher creates an isolated test-drive, installs the Git shim, and sets a+10-second snapshot deadline after test-drive clears inherited configuration.+Production's longer snapshot deadline would outlast the injected failure.+Set `BOOTSTRAP_REAL_GIT` to the absolute Git executable if it is not`/usr/bin/git`. Do not point this fixture at a normal development or productioninstance. The seeder checks the loopback host, trusted-local mode, and companydiff --git a/tests/e2e/workspace-bootstrap/launch.ts b/tests/e2e/workspace-bootstrap/launch.tsnew file mode 100644index 0000000000..c05925c76a--- /dev/null+++ b/tests/e2e/workspace-bootstrap/launch.ts@@ -0,0 +1,18 @@+import path from "node:path";+import { prepareTestDriveEnvironment, testDriveCommand } from "../../../cli/src/commands/test-drive.js";++const options = {+ harness: "codex",+ apiKeyEnv: "BOOTSTRAP_FIXTURE_KEY",+ companyName: "Workspace Recovery QA",+ browser: false,+} as const;++process.env.BOOTSTRAP_FIXTURE_KEY = "not-a-provider-key";+await prepareTestDriveEnvironment(options);+process.env.NODE_ENV = "test";+// Test-drive clears PAPERCLIP_* settings. Apply the fixture's short deadline+// after isolation; production snapshots may legitimately take much longer.+process.env.PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS = "10000";+process.env.PATH = `${path.join(import.meta.dirname, "bin")}${path.delimiter}${process.env.PATH ?? ""}`;+await testDriveCommand(options);diff --git a/tests/e2e/workspace-bootstrap/recovery.spec.ts b/tests/e2e/workspace-bootstrap/recovery.spec.tsindex f8ac233e81..884a20a8bd 100644--- a/tests/e2e/workspace-bootstrap/recovery.spec.ts+++ b/tests/e2e/workspace-bootstrap/recovery.spec.ts@@ -22,11 +22,11 @@ for (const persistent of [false, true]) {if (await announcement.isVisible()) await announcement.click();await page.getByRole("link", { name: "Tasks", exact: true }).click();await page.getByRole("button", { name: "New Task", exact: true }).last().click();- await page.getByPlaceholder("Task title (optional)", { exact: true }).fill(title);- await page.getByRole("button", { name: "Assignee", exact: true }).click();- await page.getByRole("button", { name: fixture.agentName, exact: true }).click();+ await page.getByRole("textbox", { name: "editable markdown" }).fill(title);+ await page.getByRole("button", { name: "Select assignee", exact: true }).click();+ await page.getByRole("option").filter({ hasText: fixture.agentName }).click();// Let the closing popover unmount before clicking another popover trigger.- await expect(page.getByRole("textbox", { name: "Search assignees...", includeHidden: true })).toHaveCount(0);+ await expect(page.getByRole("searchbox", { name: "Search assignees", includeHidden: true })).toHaveCount(0);// The preceding popover's focus restoration can consume the first click.await expect(async () => {if (!await page.getByRole("textbox", { name: "Search projects..." }).isVisible()) {@@ -36,15 +36,21 @@ for (const persistent of [false, true]) {}).toPass({ timeout: 10_000, intervals: [1_000] });await page.getByRole("button", { name: fixture.projectName, exact: true }).click();await expect(page.getByRole("textbox", { name: "Search projects...", includeHidden: true })).toHaveCount(0);- await page.getByRole("button", { name: "Create Task", exact: true }).click();- const taskLink = page.getByRole("complementary").getByRole("link", { name: title, exact: true });+ const created = page.waitForResponse(response =>+ response.request().method() === "POST"+ && new URL(response.url()).pathname === `/api/companies/${fixture.companyId}/issues`,+ );+ await page.getByRole("button", { name: "Create task", exact: true }).click();+ const response = await created;+ expect(response.ok(), await response.text()).toBeTruthy();+ const task = await response.json();+ expect(task.description).toBe(title);+ const taskLink = page.getByRole("complementary").getByRole("link", { name: task.title, exact: true });await expect(taskLink).toBeVisible();// Follow the UI's actual persisted link, including across task-tab state.const href = await taskLink.getAttribute("href");expect(href).toBeTruthy();await page.goto(new URL(href!, base).href);- const tasks = await api(`/companies/${fixture.companyId}/issues`);- const task = tasks.find((row: { title: string }) => row.title === title);const runs = async () => (await api(`/companies/${fixture.companyId}/heartbeat-runs`)).filter((row: { agentId: string }) => row.agentId === fixture.agentId);await expect.poll(async () => (await runs()).some((row: { errorCode: string }) => row.errorCode === "workspace_git_scan_timeout"), { timeout: 30_000 }).toBe(true);await expect(page.getByText(/Agent resumes in/)).toBeVisible();