Skip to content

The open-source app everyone uses to manage agents at work

refactor(heartbeat): extract workspace preparation (#15807)

Dotta07fee85eafda
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md
index f58ef608b0..fd665cdb6d 100644
--- a/doc/DEVELOPING.md
+++ b/doc/DEVELOPING.md
@@ -1360,6 +1360,14 @@ lifecycle and ownership callbacks update its teardown state immediately, includi
when preparation later throws. Public sandbox lifecycle and model fallback helpers
remain re-exported by `heartbeat.ts`.
+Workspace preparation is in `server/src/services/heartbeat/workspace-preparation.ts`.
+`prepareHeartbeatWorkspace` resolves and provisions the workspace, persists its
+metadata and issue binding, then acquires and realizes the execution environment.
+It preserves the immutable workspace binding used by native recovery and records
+preparation spans even when acquisition or realization fails. The executor retains
+environment selection, dispatch gates, and terminal lease cleanup. Persistence
+failures still clean newly created workspace artifacts before propagating.
+
Runtime execution is in `server/src/services/heartbeat/runtime-execution.ts`.
`executeHeartbeatRuntime` dispatches a prepared native session or legacy adapter,
captures usage, finalizes workspace operations, and releases gateway tokens and
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index 5517583e66..a0979025f2 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -8455,12 +8455,10 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
.set({ status: "paused" })
.where(eq(agents.id, agentId));
release();
- if (next!)
- await vi.waitFor(async () =>
- expect((await heartbeat.getRun(next!.id))?.status).not.toBe(
- "running",
- ),
- );
+ // Wait for the released executor and its cleanup, not the default
+ // one-second polling window. The next case must not inherit live work.
+ await heartbeat.drainActiveRunExecutions();
+ if (next!) expect((await heartbeat.getRun(next!.id))?.status).not.toBe("running");
}
},
);
@@ -12100,7 +12098,9 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const legacyReviewBefore = withLegacyReview
? await db.select().from(issues).where(eq(issues.id, legacyReviewId))
: [];
- mockAdapterExecute.mockImplementationOnce(async () => {
+ let planOnlyRunId = "";
+ mockAdapterExecute.mockImplementationOnce(async (input?: unknown) => {
+ planOnlyRunId = (input as { runId: string }).runId;
if (withLegacyReview) {
// These pre-dispatch cancellations used to satisfy both the no-comment
// and churn thresholds and suppress an otherwise valid continuation.
@@ -12124,17 +12124,23 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
await heartbeat.reconcileStrandedAssignedIssues();
await heartbeat.promoteDueScheduledRetries(new Date(Date.now() + 31_000));
await heartbeat.resumeQueuedRuns();
+ // The continuation is produced by completion; wait for that durable boundary.
+ await heartbeat.drainActiveRunExecutions();
- const livenessWake = await waitForValue(async () => {
+ expect(planOnlyRunId).not.toBe("");
+ const livenessWakes = await waitForValue(async () => {
const rows = await db
.select()
.from(agentWakeupRequests)
.where(eq(agentWakeupRequests.agentId, agentId));
- return (
- rows.find((row) => row.reason === "issue_disposition_repair") ?? null
- );
- });
- expect(livenessWake).toBeTruthy();
+ // Follow-up repairs can already exist. Match this plan-only result,
+ // rather than relying on the database's unordered row order.
+ const matching = rows.filter((row) => row.reason === "issue_disposition_repair"
+ && row.payload?.retryOfRunId === planOnlyRunId);
+ return matching.length > 0 ? matching : null;
+ });
+ expect(livenessWakes).toHaveLength(1);
+ const livenessWake = livenessWakes![0]!;
expect(livenessWake?.payload).toMatchObject({
issueId,
dispositionRepairAttempt: 1,
@@ -12143,7 +12149,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
const sourceRunId = (
livenessWake?.payload as Record<string, unknown> | null
)?.retryOfRunId;
- expect(sourceRunId).toBeTruthy();
+ expect(sourceRunId).toBe(planOnlyRunId);
const sourceRun = await db
.select()
.from(heartbeatRuns)
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index ae86b02f87..576975ce90 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -1,3 +1,4 @@
+import { prepareHeartbeatWorkspace } from "./heartbeat/workspace-preparation.js";
import { executeHeartbeatRuntime, NativeSessionResumeScheduledError, NativeWorkspaceFinalizeScheduledError } from "./heartbeat/runtime-execution.js";
import { selectHeartbeatRuntime } from "./heartbeat/runtime-selection.js";
import { createHeartbeatRunCompletion } from "./heartbeat/run-completion.js";
@@ -145,19 +146,6 @@ import {
readConfiguredModelFromAdapterConfig,
resolveTaskSessionConfigFreshness,
stripPaperclipSessionMetadataFromSessionParams,
- resolveWorkspaceAfterLowTrustPreflight,
- WorkspaceValidationFailure,
- stripHostWorkspaceProvisionForLowTrustSandbox,
- assertGitWorktreeBaseWorkspaceReady,
- buildEffectiveRunWorkspaceConfigMetadata,
- resolveExecutionWorkspaceConfigFreshness,
- resolveExecutionWorkspaceReuseProvisioningPolicy,
- provisionExecutionWorkspaceForFreshnessDecision,
- mergeExecutionWorkspaceMetadataForPersistence,
- resolveExecutionWorkspaceBranchOwnership,
- reconcileReusedExecutionWorkspaceProjectWorkspaceId,
- recordWorkspaceConfigFreshnessOperation,
- prepareProjectRepositoryWorkspaces,
resolveRuntimeSessionParamsForWorkspace,
buildRunWorkspaceHints,
buildReferencedProjectRunObservability,
@@ -244,8 +232,6 @@ import { agentIdentityService, supportsManagedAgentIdentity } from "./agent-iden
import { prepareConnectionInstructionDelivery } from "./connection-instructions.js";
import { resolveAssignedConnectionInstructionsForRun } from "./native-runtime/assigned-mcp-tools.js";
-import { resolvePaperclipInstanceRoot } from "../home-paths.js";
-
import {
startupCancellationFence,
} from "./native-runtime/native-cancellation-request.js";
@@ -341,7 +327,6 @@ import {
} from "node:crypto";
import {
and,
- asc,
desc,
eq,
exists,
@@ -356,7 +341,6 @@ import {
CHAT_PROVIDERS,
ISSUE_DISPOSITION_REPAIR_RETRY_REASON,
isEnvironmentDriverSupportedForAdapter,
- type ExecutionWorkspace,
} from "@paperclipai/shared";
import {
agents,
@@ -372,14 +356,9 @@ import {
projectWorkspaces,
} from "@paperclipai/db";
import { HttpError, notFound } from "../errors.js";
-import {
- getStartupTracer,
-} from "../instrumentation.js";
-import { createHostDuplexObservabilityRecorder } from "./duplex-observability-recorder.js";
-import { incrementToolRuntimeMetricCounter } from "./tool-runtime-metrics.js";
+
import { logger } from "../middleware/logger.js";
import {
- createGitRemoteAuthProvider,
resolveManagedGitHubIdentitySelection,
scrubGitCredentialText,
} from "./git-credentials.js";
@@ -402,7 +381,6 @@ import {
NativeCancellationPendingRecoveryError,
NativeControllerDetachedForRestartError,
prepareNativeWorkspaceSync,
- readNativeWorkspaceSyncReference,
type NativeRestartRecoveryClaim,
resolveHeartbeatNativeRuntimeMode,
} from "./native-runtime/index.js";
@@ -444,11 +422,10 @@ import {
} from "./native-runtime/native-runner-ownership.js";
import {
findNativeChatWorkspaceScope,
- materializeNativeChatTaskRoot,
nativeChatWorkspaceCwd,
nativeChatWorkspaceMatches,
} from "./native-runtime/native-chat-workspace.js";
-import { materializeIsolatedTaskDirectory, shouldUseIsolatedTaskDirectory } from "./isolated-task-directory.js";
+import { shouldUseIsolatedTaskDirectory } from "./isolated-task-directory.js";
import { trackAgentFirstHeartbeat } from "@paperclipai/shared/telemetry";
import { getTelemetryClient } from "../telemetry.js";
import {
@@ -481,21 +458,13 @@ import {
buildWorkspaceReadyComment,
buildWorkspaceReadyMetadata,
buildWorkspaceReadyPresentation,
- cleanupExecutionWorkspaceArtifacts,
- ensurePersistedExecutionWorkspaceAvailable,
ensureRuntimeServicesForRun,
persistAdapterManagedRuntimeServices,
- realizeExecutionWorkspace,
releaseRuntimeServicesForRun,
- type ExecutionWorkspaceInput,
type RealizedExecutionWorkspace,
type RuntimeServiceRef,
} from "./workspace-runtime.js";
-import {
- readManagedWorktreeInstanceOwnership,
- WORKTREE_INSTANCE_ROOT_METADATA_KEY,
-} from "./workspace-instance-cleanup.js";
import { issueService } from "./issues.js";
import {
failRunnerGoalAction,
@@ -521,9 +490,7 @@ import {
import {
isProcessGroupAlive,
} from "./local-service-supervisor.js";
-import {
- isRuntimeOwnedGitBranch,
-} from "./execution-workspace-branch-ownership.js";
+
import {
HEARTBEAT_RUN_SCRATCH_MARKER,
buildHeartbeatRunScratchEnv,
@@ -535,10 +502,8 @@ import {
applyDefaultIsolatedExecutionWorkspacePolicy,
buildExecutionWorkspaceAdapterConfig,
gateProjectExecutionWorkspacePolicy,
- issueExecutionWorkspaceModeForPersistedWorkspace,
parseIssueExecutionWorkspaceSettings,
parseProjectExecutionWorkspacePolicy,
- resolveEffectiveWorkspaceStrategyType,
resolveExecutionWorkspaceEnvironmentId,
resolveExecutionWorkspaceMode,
resolveSharedWorkspaceConcurrency,
@@ -549,7 +514,6 @@ import {
resolveWorktreeRunExecutionActivation,
} from "./instance-settings.js";
import {
- evaluateExecutionAllowlist,
isExecutionForcedToKubernetes,
} from "./execution-allowlist.js";
@@ -3692,862 +3656,78 @@ export function heartbeatService(
if (handoff) context.paperclipFreshSessionHandoffMarkdown = handoff;
}
const {
- selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,
- workspace: resolvedWorkspace,
- } = await resolveWorkspaceAfterLowTrustPreflight({
- db,
- trustPreset,
- isolatedWorkspacesEnabled,
- effectiveExecutionWorkspaceMode,
- issue: issueRef
- ? {
- companyId: agent.companyId,
- id: issueRef.id,
- projectId: issueRef.projectId,
- }
- : null,
- resolveSelectedEnvironmentDriver: async () => {
- const preflightEnvironment = await envOrchestrator.resolveEnvironment(
- {
- companyId: agent.companyId,
- selectedEnvironmentId,
- localEnvironmentId: localEnvironment.id,
- },
- );
- return preflightEnvironment.driver;
- },
- resolveWorkspace: async () => {
- if (isDotRun) {
- // This is private controller storage, never a provider filesystem.
- // v6 projects workspace.access=none and cwd=null to Dot.
- const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "dot-controllers", agent.companyId, run.id);
- await fs.mkdir(cwd, { recursive: true, mode: 0o700 });
- return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null,
- workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] };
- }
- if (useIsolatedTaskDirectory && issueRef) {
- const cwd = await materializeIsolatedTaskDirectory({
- companyId: agent.companyId,
- issueId: issueRef.id,
- });
- if (reusableExistingExecutionWorkspace && (
- reusableExistingExecutionWorkspace.companyId !== agent.companyId ||
- reusableExistingExecutionWorkspace.projectId !== issueRef.projectId ||
- reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id ||
- reusableExistingExecutionWorkspace.mode !== "isolated_workspace" ||
- reusableExistingExecutionWorkspace.strategyType !== "project_primary" ||
- reusableExistingExecutionWorkspace.cwd !== cwd
- )) {
- throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", {
- workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id },
- });
- }
- return resolveWorkspaceForRun(agent, context, previousSessionParams, {
- executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null,
- anchorWorkspace: {
- cwd,
- source: "task_session",
- projectId: issueRef.projectId,
- workspaceId: null,
- repoUrl: null,
- repoRef: null,
- workspaceHints: [],
- warnings: [],
- baseCwdFallback: false,
- materializationFailures: [],
- },
- });
- }
- if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) {
- const cwd = await materializeNativeChatTaskRoot(
- nativeChatWorkspaceScope,
- );
- return {
- cwd,
- source: "task_session" as const,
- projectId: null,
- workspaceId: null,
- repoUrl: null,
- repoRef: null,
- workspaceHints: [],
- warnings: [],
- baseCwdFallback: false,
- materializationFailures: [],
- additionalWorkspaces: [],
- referencedProjectFailures: [],
- };
- }
- const workspace = await resolveWorkspaceForRun(
- agent,
- context,
- previousSessionParams,
- {
- useProjectWorkspace:
- requestedExecutionWorkspaceMode !== "agent_default",
- anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"
- ? await resolveReusedGitWorkspaceAnchor({
- agent,
- workspace: reusableExistingExecutionWorkspace,
- responsibleUserId,
- immutableNativeBinding: Boolean(nativeRecoveryExecutionWorkspaceId),
- projectId: nativeRecoveryExecutionWorkspaceId
- ? reusableExistingExecutionWorkspace.projectId
- : issueRef?.projectId ?? readNonEmptyString(context.projectId),
- explicitProjectWorkspaceId: nativeRecoveryExecutionWorkspaceId
- ? reusableExistingExecutionWorkspace.projectWorkspaceId
- : readNonEmptyString(context.projectWorkspaceId),
- issueId,
- runId: run.id,
- })
- : undefined,
- // Thread the selected environment driver so run-workspace resolution can tell a local
- // target from a remote one, and a confined sandbox target from an unconfined remote
- // target. A remote run resolves referenced projects only for the confined sandbox
- // transport with the remote flag on. This never changes the anchor workspace.
- executionEnvironmentDriver:
- selectedEnvironmentForConfig?.driver ?? null,
- },
- );
- // Additional referenced projects are a separate trusted Board
- // capability, not extra readable roots for an external conversation.
- return nativeChatWorkspaceScope
- ? {
- ...workspace,
- additionalWorkspaces: [],
- referencedProjectFailures: [],
- }
- : workspace;
+ resolvedWorkspace,
+ hostExecutionWorkspaceConfig,
+ latestWorkspaceConfigMetadata,
+ workspaceConfigFreshness,
+ workspaceOperationRecorder,
+ executionWorkspace,
+ reusedExecutionWorkspace,
+ resolvedWorkspaceReusePolicy,
+ persistedExecutionWorkspace,
+ projectRepositoryPaths,
+ environmentAcquireStartedAtMs,
+ selectedEnvironment,
+ activeEnvironmentLease,
+ realizationResult,
+ environmentRealizeEndedAtMs,
+ workspaceRealization,
+ executionTarget,
+ workspaceRestoreSource,
+ } = await prepareHeartbeatWorkspace(db, {
+ run, agent, controllerLease, nativeRunnerPreparationSpans,
+ task: {
+ issueRef,
+ issueId,
+ context,
+ executionProjectId,
+ responsibleUserId,
+ previousSessionParams,
},
- });
- const hostExecutionWorkspaceConfig = isDotRun ? {} :
- stripHostWorkspaceProvisionForLowTrustSandbox({
- config: mergedConfig,
+ policy: {
trustPreset,
- selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,
- });
- const executionWorkspaceBase = {
- baseCwd: resolvedWorkspace.cwd,
- source: resolvedWorkspace.source,
- projectId: resolvedWorkspace.projectId,
- workspaceId: resolvedWorkspace.workspaceId,
- repoUrl: resolvedWorkspace.repoUrl,
- repoRef: resolvedWorkspace.repoRef,
- additionalWorkspaces: resolvedWorkspace.additionalWorkspaces,
- } satisfies ExecutionWorkspaceInput;
- await assertGitWorktreeBaseWorkspaceReady({
- requestedExecutionWorkspaceMode,
- config: hostExecutionWorkspaceConfig,
- issue: issueRef,
- base: executionWorkspaceBase,
- anchor: {
- baseCwdFallback: resolvedWorkspace.baseCwdFallback,
- materializationFailures: resolvedWorkspace.materializationFailures,
- localPathOnlyWorkspace: resolvedWorkspace.localPathOnlyWorkspace,
+ isolatedWorkspacesEnabled,
+ effectiveExecutionWorkspaceMode,
+ requestedExecutionWorkspaceMode,
+ useIsolatedTaskDirectory,
+ nativeChatWorkspaceScope,
+ projectExecutionWorkspacePolicy,
+ issueExecutionWorkspaceSettings,
+ environmentExecutionWorkspaceSettings,
+ executionPolicy,
},
- });
- const workspaceStrategyForFingerprint = parseObject(
- hostExecutionWorkspaceConfig.workspaceStrategy,
- );
- const workspaceStrategyFingerprintValue =
- Object.keys(workspaceStrategyForFingerprint).length > 0
- ? workspaceStrategyForFingerprint
- : null;
- const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType(
- requestedExecutionWorkspaceMode,
- hostExecutionWorkspaceConfig,
- );
- const selectedEnvironmentConfigForFingerprint = parseObject(
- selectedEnvironmentForConfig?.config,
- );
- const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig
- ? {
- selectionSource: environmentResolution.source,
- selectedEnvironmentId,
- driver: selectedEnvironmentForConfig.driver,
- provider: readNonEmptyString(
- selectedEnvironmentConfigForFingerprint.provider,
- ),
- config: selectedEnvironmentForConfig.config,
- configRevisionAt:
- selectedEnvironmentForConfig.updatedAt instanceof Date
- ? selectedEnvironmentForConfig.updatedAt.toISOString()
- : (selectedEnvironmentForConfig.updatedAt ?? null),
- executionPolicy,
- }
- : null;
- const workspaceRealizationFingerprint = {
- environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
- environmentProvider: readNonEmptyString(
- selectedEnvironmentConfigForFingerprint.provider,
- ),
- trustPreset: trustPreset.kind,
- lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver,
- };
- const workspaceFreshnessSource = resolvedWorkspace.freshnessSource ?? executionWorkspaceBase;
- const latestWorkspaceConfigMetadata =
- buildEffectiveRunWorkspaceConfigMetadata({
- mode: requestedExecutionWorkspaceMode,
- projectId: workspaceFreshnessSource.projectId,
- projectWorkspaceId: workspaceFreshnessSource.workspaceId,
- strategyType: latestWorkspaceStrategyType,
- workspaceStrategy: workspaceStrategyFingerprintValue,
- repoUrl: workspaceFreshnessSource.repoUrl,
- repoRef:
- readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ??
- workspaceFreshnessSource.repoRef,
+ environment: {
+ selectedEnvironmentId,
+ localEnvironment,
+ selectedEnvironmentForConfig,
+ environmentResolution,
+ resolvedInstanceSettings,
+ },
+ config: {
+ mergedConfig,
configSnapshot,
- environment: workspaceEnvironmentFingerprint,
- realization: workspaceRealizationFingerprint,
secretManifest,
- });
- const inferredExistingWorkspaceConfigMetadata =
- reusableExistingExecutionWorkspace
- ? buildEffectiveRunWorkspaceConfigMetadata({
- mode: issueExecutionWorkspaceModeForPersistedWorkspace(
- reusableExistingExecutionWorkspace.mode,
- ),
- projectId: reusableExistingExecutionWorkspace.projectId,
- projectWorkspaceId:
- reusableExistingExecutionWorkspace.projectWorkspaceId,
- strategyType: reusableExistingExecutionWorkspace.strategyType,
- workspaceStrategy: workspaceStrategyFingerprintValue
- ? {
- ...workspaceStrategyFingerprintValue,
- type: reusableExistingExecutionWorkspace.strategyType,
- ...(reusableExistingExecutionWorkspace.baseRef
- ? { baseRef: reusableExistingExecutionWorkspace.baseRef }
- : {}),
- }
- : { type: reusableExistingExecutionWorkspace.strategyType },
- repoUrl: reusableExistingExecutionWorkspace.repoUrl,
- repoRef: reusableExistingExecutionWorkspace.baseRef,
- configSnapshot: reusableExistingExecutionWorkspace.config,
- environment: workspaceEnvironmentFingerprint,
- realization: workspaceRealizationFingerprint,
- secretManifest,
- evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt,
- })
- : null;
- const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness(
- {
- hasExistingWorkspace:
- requestedShouldReuseExisting &&
- Boolean(reusableExistingExecutionWorkspace),
- existingWorkspaceMetadata:
- reusableExistingExecutionWorkspace?.metadata ?? null,
- inferredMetadata: inferredExistingWorkspaceConfigMetadata,
- nextMetadata: latestWorkspaceConfigMetadata,
},
- );
- const workspaceReuseProvisioningPolicy =
- resolveExecutionWorkspaceReuseProvisioningPolicy({
+ reuse: {
requestedShouldReuseExisting,
- workspaceConfigFreshness,
- });
- const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
- companyId: agent.companyId,
- heartbeatRunId: run.id,
- executionWorkspaceId:
- workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace
- ? workspaceReuseRequest.requestedExecutionWorkspaceId
- : null,
- issueId,
- });
- // The run-scoped provider resolves the active identity at each Git operation,
- // including base-ref refreshes, workspace realization, and restore.
- const workspaceGitAuthProvider = createGitRemoteAuthProvider(
- db,
- agent.companyId,
- {
- issueId,
- heartbeatRunId: run.id,
- responsibleUserId: run.responsibleUserId,
- agentId: agent.id,
+ existingExecutionWorkspace,
+ reusableExistingExecutionWorkspace,
+ workspaceReuseRequest,
+ nativeRecoveryExecutionWorkspaceId,
+ persistedNativeExecutionInput,
+ isDotRun,
+ runOptions,
},
- );
- const {
- executionWorkspace,
- reusedExecutionWorkspace,
- policy: resolvedWorkspaceReusePolicy,
- } = isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
- {
- requestedShouldReuseExisting,
- existingExecutionWorkspaceId:
- workspaceReuseRequest.requestedExecutionWorkspaceId,
- issueRef,
- runId: run.id,
- workspaceConfigFreshness,
- restoreExistingWorkspace: reusableExistingExecutionWorkspace
- ? () =>
- ensurePersistedExecutionWorkspaceAvailable({
- db,
- base: executionWorkspaceBase,
- workspace: {
- id: reusableExistingExecutionWorkspace.id,
- mode: reusableExistingExecutionWorkspace.mode,
- strategyType:
- reusableExistingExecutionWorkspace.strategyType,
- cwd: reusableExistingExecutionWorkspace.cwd,
- providerRef: reusableExistingExecutionWorkspace.providerRef,
- projectId: reusableExistingExecutionWorkspace.projectId,
- projectWorkspaceId:
- reusableExistingExecutionWorkspace.projectWorkspaceId,
- repoUrl: reusableExistingExecutionWorkspace.repoUrl,
- baseRef: reusableExistingExecutionWorkspace.baseRef,
- branchName: reusableExistingExecutionWorkspace.branchName,
- metadata:
- reusableExistingExecutionWorkspace.metadata as Record<
- string,
- unknown
- > | null,
- config: {
- provisionCommand:
- configSnapshot?.provisionCommand ??
- reusableExistingExecutionWorkspace.config
- ?.provisionCommand ??
- projectExecutionWorkspacePolicy?.workspaceStrategy
- ?.provisionCommand ??
- null,
- runtimeProvisionCommand:
- configSnapshot?.runtimeProvisionCommand ??
- reusableExistingExecutionWorkspace.config
- ?.runtimeProvisionCommand ??
- projectExecutionWorkspacePolicy?.workspaceStrategy
- ?.runtimeProvisionCommand ??
- null,
- },
- },
- issue: issueRef,
- agent: {
- id: agent.id,
- name: agent.name,
- companyId: agent.companyId,
- },
- heartbeatRunId: run.id,
- enableWorkspaceBranchReconcileForward:
- resolvedInstanceSettings.experimental
- .enableWorkspaceBranchReconcileForward,
- enableWorkspaceDirtyQuarantineRepair:
- resolvedInstanceSettings.experimental
- .enableWorkspaceDirtyQuarantineRepair,
- recorder: workspaceOperationRecorder,
- resolveGitAuth: workspaceGitAuthProvider,
- })
- : null,
- realizeWorkspace: () =>
- realizeExecutionWorkspace({
- db,
- base: executionWorkspaceBase,
- config: hostExecutionWorkspaceConfig,
- issue: issueRef,
- agent: {
- id: agent.id,
- name: agent.name,
- companyId: agent.companyId,
- },
- recordedBranchOwnership:
- existingExecutionWorkspace?.status !== "archived" &&
- existingExecutionWorkspace?.branchName
- ? {
- branchName: existingExecutionWorkspace.branchName,
- createdByRuntime: isRuntimeOwnedGitBranch(
- existingExecutionWorkspace.metadata,
- ),
- }
- : null,
- heartbeatRunId: run.id,
- enableWorkspaceBranchReconcileForward:
- resolvedInstanceSettings.experimental
- .enableWorkspaceBranchReconcileForward,
- enableWorkspaceDirtyQuarantineRepair:
- resolvedInstanceSettings.experimental
- .enableWorkspaceDirtyQuarantineRepair,
- recorder: workspaceOperationRecorder,
- resolveGitAuth: workspaceGitAuthProvider,
- }),
+ services: {
+ envOrchestrator,
+ executionWorkspacesSvc,
+ workspaceOperationsSvc,
+ issuesSvc,
+ resolveWorkspaceForRun,
+ resolveReusedGitWorkspaceAnchor,
+ appendRunEvent,
},
- );
- const resolvedProjectId =
- executionWorkspace.projectId ??
- issueRef?.projectId ??
- executionProjectId ??
- null;
- const resolvedProjectWorkspaceId =
- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"
- ? reusableExistingExecutionWorkspace.projectWorkspaceId
- : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
- let persistedExecutionWorkspace: ExecutionWorkspace | null = null;
- let issueExecutionWorkspaceIdForRun =
- issueRef?.executionWorkspaceId ?? null;
- let issueProjectWorkspaceIdForRun = issueRef?.projectWorkspaceId ?? null;
- let issueExecutionWorkspacePreferenceForRun =
- issueRef?.executionWorkspacePreference ?? null;
- let issueExecutionWorkspaceModeForRun =
- issueExecutionWorkspaceSettings?.mode ?? null;
- const warmReusableExecutionWorkspace =
- selectedEnvironmentForConfig?.driver === "sandbox" &&
- selectedEnvironmentConfigForFingerprint.reuseLease === true &&
- selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm";
- // Native provider checkpoints bind to the workspace row, including ordinary
- // local shared workspaces. Persist that binding independently of the opt-in
- // isolated-workspace UI, just as warm sandbox continuity already does.
- const nativeSharedWorkspace = agent.adapterType === "paperclip_runner" &&
- requestedExecutionWorkspaceMode === "shared_workspace";
- const bindIssueToPersistedExecutionWorkspace = async (
- workspace: ExecutionWorkspace | null,
- ) => {
- if (!issueId || !workspace || nativeRecoveryExecutionWorkspaceId) {
- return;
- }
- const nextIssueWorkspaceMode =
- issueExecutionWorkspaceModeForPersistedWorkspace(workspace.mode) ??
- "agent_default";
- const shouldSwitchIssueToExistingWorkspace =
- issueRef?.executionWorkspacePreference === "reuse_existing" ||
- requestedExecutionWorkspaceMode === "isolated_workspace" ||
- requestedExecutionWorkspaceMode === "operator_branch" ||
- warmReusableExecutionWorkspace || nativeSharedWorkspace;
- const nextIssuePatch: Record<string, unknown> = {};
- if (issueExecutionWorkspaceIdForRun !== workspace.id) {
- nextIssuePatch.executionWorkspaceId = workspace.id;
- }
- if (
- resolvedProjectWorkspaceId &&
- issueProjectWorkspaceIdForRun !== resolvedProjectWorkspaceId
- ) {
- nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;
- }
- if (
- shouldSwitchIssueToExistingWorkspace &&
- (issueExecutionWorkspacePreferenceForRun !== "reuse_existing" ||
- issueExecutionWorkspaceModeForRun !== nextIssueWorkspaceMode)
- ) {
- nextIssuePatch.executionWorkspacePreference = "reuse_existing";
- nextIssuePatch.executionWorkspaceSettings = {
- ...(issueExecutionWorkspaceSettings ?? {}),
- mode: nextIssueWorkspaceMode,
- };
- }
- if (Object.keys(nextIssuePatch).length > 0) {
- await issuesSvc.update(
- issueId,
- { ...nextIssuePatch, companyGuard: agent.companyId },
- db,
- undefined,
- undefined,
- { bindRuntimeSharedWorkspace: (warmReusableExecutionWorkspace || nativeSharedWorkspace) && workspace.mode === "shared_workspace" },
- );
- issueExecutionWorkspaceIdForRun = workspace.id;
- issueProjectWorkspaceIdForRun =
- resolvedProjectWorkspaceId ?? issueProjectWorkspaceIdForRun;
- if (shouldSwitchIssueToExistingWorkspace) {
- issueExecutionWorkspacePreferenceForRun = "reuse_existing";
- issueExecutionWorkspaceModeForRun = nextIssueWorkspaceMode;
- }
- }
- };
- const baseExecutionWorkspaceMetadata =
- mergeExecutionWorkspaceMetadataForPersistence({
- existingMetadata:
- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace
- ? (reusableExistingExecutionWorkspace?.metadata ?? null)
- : null,
- source: executionWorkspace.source,
- // Attaching a new worktree to a pre-existing branch reports a fresh
- // workspace, but must not make cleanup own the operator's branch.
- createdByRuntime:
- resolveExecutionWorkspaceBranchOwnership(executionWorkspace),
- strategyType: executionWorkspace.strategy,
- configSnapshot,
- shouldReuseExisting:
- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace,
- shouldRefreshConfigSnapshot:
- resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
- workspaceConfigMetadata:
- resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata
- ? latestWorkspaceConfigMetadata
- : null,
- baseRef: executionWorkspace.repoRef,
- baseRefSha: executionWorkspace.baseRefSha ?? null,
- });
- let persistedWorktreeInstanceRoot =
- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&
- typeof reusableExistingExecutionWorkspace?.metadata?.[
- WORKTREE_INSTANCE_ROOT_METADATA_KEY
- ] === "string"
- ? reusableExistingExecutionWorkspace.metadata[
- WORKTREE_INSTANCE_ROOT_METADATA_KEY
- ]
- : null;
- if (
- !persistedWorktreeInstanceRoot &&
- executionWorkspace.strategy === "git_worktree" &&
- executionWorkspace.worktreePath
- ) {
- try {
- persistedWorktreeInstanceRoot =
- (
- await readManagedWorktreeInstanceOwnership(
- executionWorkspace.worktreePath,
- )
- )?.instanceRoot ?? null;
- } catch (error) {
- logger.warn(
- {
- runId: run.id,
- issueId,
- executionWorkspaceCwd: executionWorkspace.cwd,
- error: error instanceof Error ? error.message : String(error),
- },
- "Could not record managed worktree instance ownership",
- );
- }
- }
- const nextExecutionWorkspaceMetadata = {
- ...baseExecutionWorkspaceMetadata,
- ...(persistedWorktreeInstanceRoot
- ? {
- [WORKTREE_INSTANCE_ROOT_METADATA_KEY]:
- persistedWorktreeInstanceRoot,
- }
- : {}),
- };
- const pendingForwardBranchReconcile =
- executionWorkspace.pendingForwardBranchReconcile ?? null;
- const branchNameForInitialPersistence =
- pendingForwardBranchReconcile?.recordedBranchName ??
- executionWorkspace.branchName;
- try {
- persistedExecutionWorkspace =
- resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&
- reusableExistingExecutionWorkspace
- ? await executionWorkspacesSvc.update(
- reusableExistingExecutionWorkspace.id,
- {
- cwd: executionWorkspace.cwd,
- repoUrl: executionWorkspace.repoUrl,
- baseRef: executionWorkspace.repoRef,
- branchName: branchNameForInitialPersistence,
- providerType:
- executionWorkspace.strategy === "git_worktree"
- ? "git_worktree"
- : "local_fs",
- providerRef: executionWorkspace.worktreePath,
- status: "active",
- lastUsedAt: new Date(),
- metadata: nextExecutionWorkspaceMetadata,
- projectWorkspaceId:
- reconcileReusedExecutionWorkspaceProjectWorkspaceId(
- reusableExistingExecutionWorkspace.projectWorkspaceId,
- resolvedProjectWorkspaceId,
- ),
- },
- )
- : resolvedProjectId
- ? await executionWorkspacesSvc.create({
- companyId: agent.companyId,
- projectId: resolvedProjectId,
- projectWorkspaceId: resolvedProjectWorkspaceId,
- sourceIssueId: issueRef?.id ?? null,
- mode:
- requestedExecutionWorkspaceMode === "isolated_workspace"
- ? "isolated_workspace"
- : requestedExecutionWorkspaceMode === "operator_branch"
- ? "operator_branch"
- : requestedExecutionWorkspaceMode === "agent_default"
- ? "adapter_managed"
- : "shared_workspace",
- strategyType:
- executionWorkspace.strategy === "git_worktree"
- ? "git_worktree"
- : "project_primary",
- name:
- branchNameForInitialPersistence ??
- issueRef?.identifier ??
- `workspace-${agent.id.slice(0, 8)}`,
- status: "active",
- cwd: executionWorkspace.cwd,
- repoUrl: executionWorkspace.repoUrl,
- baseRef: executionWorkspace.repoRef,
- branchName: branchNameForInitialPersistence,
- providerType:
- executionWorkspace.strategy === "git_worktree"
- ? "git_worktree"
- : "local_fs",
- providerRef: executionWorkspace.worktreePath,
- lastUsedAt: new Date(),
- openedAt: new Date(),
- metadata: nextExecutionWorkspaceMetadata,
- })
- : null;
- } catch (error) {
- if (executionWorkspace.created) {
- try {
- await cleanupExecutionWorkspaceArtifacts({
- workspace: {
- id:
- reusableExistingExecutionWorkspace?.id ??
- workspaceReuseRequest.requestedExecutionWorkspaceId ??
- `transient-${run.id}`,
- cwd: executionWorkspace.cwd,
- providerType:
- executionWorkspace.strategy === "git_worktree"
- ? "git_worktree"
- : "local_fs",
- providerRef: executionWorkspace.worktreePath,
- branchName: executionWorkspace.branchName,
- repoUrl: executionWorkspace.repoUrl,
- baseRef: executionWorkspace.repoRef,
- projectId: resolvedProjectId,
- projectWorkspaceId: resolvedProjectWorkspaceId,
- sourceIssueId: issueRef?.id ?? null,
- metadata: nextExecutionWorkspaceMetadata,
- },
- projectWorkspace: {
- cwd: resolvedWorkspace.cwd,
- cleanupCommand: null,
- },
- cleanupCommand: configSnapshot?.cleanupCommand ?? null,
- teardownCommand:
- configSnapshot?.teardownCommand ??
- projectExecutionWorkspacePolicy?.workspaceStrategy
- ?.teardownCommand ??
- null,
- recorder: workspaceOperationRecorder,
- });
- } catch (cleanupError) {
- logger.warn(
- {
- runId: run.id,
- issueId,
- executionWorkspaceCwd: executionWorkspace.cwd,
- cleanupError:
- cleanupError instanceof Error
- ? cleanupError.message
- : String(cleanupError),
- },
- "Failed to cleanup realized execution workspace after persistence failure",
- );
- }
- }
- throw error;
- }
- await workspaceOperationRecorder.attachExecutionWorkspaceId(
- persistedExecutionWorkspace?.id ?? null,
- );
- await recordWorkspaceConfigFreshnessOperation({
- recorder: workspaceOperationRecorder,
- runId: run.id,
- decision: workspaceConfigFreshness,
- hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace),
- reuseRequested: requestedShouldReuseExisting,
- workspaceReused: Boolean(reusedExecutionWorkspace),
- configSnapshotRefreshed:
- resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
- previousWorkspaceId:
- workspaceReuseRequest.requestedExecutionWorkspaceId,
- activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,
- });
- if (
- reusableExistingExecutionWorkspace &&
- persistedExecutionWorkspace &&
- reusableExistingExecutionWorkspace.id !==
- persistedExecutionWorkspace.id &&
- reusableExistingExecutionWorkspace.status === "active"
- ) {
- await executionWorkspacesSvc.update(
- reusableExistingExecutionWorkspace.id,
- {
- status: "idle",
- cleanupReason: null,
- },
- );
- }
- await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
- const projectRepositoryPaths: string[] = [];
- if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) {
- const repositoryRows = await db.select().from(projectWorkspaces).where(and(
- eq(projectWorkspaces.companyId, agent.companyId),
- eq(projectWorkspaces.projectId, executionWorkspace.projectId),
- )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id));
- const repositories = await prepareProjectRepositoryWorkspaces({
- cwd: executionWorkspace.cwd,
- anchorRepoUrl: executionWorkspace.repoUrl,
- workspaces: repositoryRows,
- resolveGitAuth: workspaceGitAuthProvider,
- });
- const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd]));
- projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd)));
- if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd);
- resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({
- ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd,
- }));
- }
- if (persistedExecutionWorkspace) {
- context.executionWorkspaceId = persistedExecutionWorkspace.id;
- await db
- .update(heartbeatRuns)
- .set({
- contextSnapshot: context,
- updatedAt: new Date(),
- })
- .where(eq(heartbeatRuns.id, run.id));
- }
- const environmentAcquireStartedAtMs = Date.now();
- let acquiredEnvironment: Awaited<
- ReturnType<typeof envOrchestrator.acquireForRun>
- >;
- try {
- await controllerLease.assertOwned();
- const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner"
- ? runOptions.nativeRestartRecovery : null;
- const recoveryWorkspace = remoteRecovery
- ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null;
- if (remoteRecovery && (!recoveryWorkspace || remoteRecovery.runId !== run.id ||
- recoveryWorkspace.providerLeaseId !== remoteRecovery.remote.providerLeaseId ||
- recoveryWorkspace.remoteCwd !== remoteRecovery.remote.remoteCwd)) {
- throw new Error("native_remote_recovery_lease_mismatch");
- }
- acquiredEnvironment = await envOrchestrator.acquireForRun({
- companyId: agent.companyId,
- selectedEnvironmentId,
- localEnvironmentId: localEnvironment.id,
- adapterType: agent.adapterType,
- adapterConfig: parseObject(agent.adapterConfig),
- admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode,
- issueId: issueId ?? null,
- heartbeatRunId: run.id,
- agentId: agent.id,
- persistedExecutionWorkspace,
- executionWorkspaceSettings: environmentExecutionWorkspaceSettings,
- ...(remoteRecovery && recoveryWorkspace ? { reattachRemoteLease: {
- leaseId: recoveryWorkspace.leaseId,
- providerLeaseId: remoteRecovery.remote.providerLeaseId,
- remoteCwd: remoteRecovery.remote.remoteCwd,
- } } : {}),
- });
- await controllerLease.assertOwned();
- nativeRunnerPreparationSpans.push({
- name: "environment.acquire",
- parentName: "task.run",
- startedAtMs: environmentAcquireStartedAtMs,
- endedAtMs: Date.now(),
- attributes: { adapter: agent.adapterType },
- });
- } catch (error) {
- nativeRunnerPreparationSpans.push({
- name: "environment.acquire",
- parentName: "task.run",
- startedAtMs: environmentAcquireStartedAtMs,
- endedAtMs: Date.now(),
- outcome: "failed",
- attributes: { adapter: agent.adapterType },
- });
- throw error;
- }
- const selectedEnvironment = acquiredEnvironment.environment;
- // Defense-in-depth: re-check the actually-acquired environment against the
- // execution allowlist. Even if selection were bypassed, a denied (local/ssh/
- // non-k8s) environment FAILS the run here rather than executing untrusted.
- const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, {
- driver: selectedEnvironment.driver,
- provider:
- typeof selectedEnvironment.config?.provider === "string"
- ? selectedEnvironment.config.provider
- : null,
});
- if (!allowlistDecision.allowed) {
- logger.error(
- {
- runId: run.id,
- issueId,
- agentId: agent.id,
- environmentId: selectedEnvironment.id,
- deniedDriver: allowlistDecision.deniedDriver,
- deniedProvider: allowlistDecision.deniedProvider,
- },
- "Execution allowlist denied the resolved environment; failing run",
- );
- throw new Error(allowlistDecision.reason);
- }
- let activeEnvironmentLease = {
- environment: acquiredEnvironment.environment,
- lease: acquiredEnvironment.lease,
- leaseContext: acquiredEnvironment.leaseContext,
- };
- const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder(
- {
- tracer: getStartupTracer(),
- incrementCounter: (metric) => {
- void incrementToolRuntimeMetricCounter(db, {
- companyId: run.companyId,
- metric,
- }).catch(() => {});
- },
- emitTransportEvent: (event) => {
- void (async () => {
- await appendRunEvent(run, {
- eventType: event.name,
- stream: "system",
- level: event.dimensions.outcome === "error" ? "warn" : "info",
- payload: { ...event.dimensions },
- });
- })().catch(() => {});
- },
- },
- );
- const environmentRealizeStartedAtMs = Date.now();
- let realizationResult: Awaited<
- ReturnType<typeof envOrchestrator.realizeForRun>
- >;
- try {
- realizationResult = await envOrchestrator.realizeForRun({
- environment: selectedEnvironment,
- lease: activeEnvironmentLease.lease,
- adapterType: agent.adapterType,
- companyId: agent.companyId,
- issueId: issueId ?? null,
- heartbeatRunId: run.id,
- executionWorkspace,
- effectiveExecutionWorkspaceMode,
- persistedExecutionWorkspace,
- duplexObservabilityRecorder,
- });
- nativeRunnerPreparationSpans.push({
- name: "environment.workspace.realize",
- parentName: "task.run",
- startedAtMs: environmentRealizeStartedAtMs,
- endedAtMs: Date.now(),
- attributes: { driver: selectedEnvironment.driver },
- });
- } catch (error) {
- nativeRunnerPreparationSpans.push({
- name: "environment.workspace.realize",
- parentName: "task.run",
- startedAtMs: environmentRealizeStartedAtMs,
- endedAtMs: Date.now(),
- outcome: "failed",
- attributes: { driver: selectedEnvironment.driver },
- });
- throw error;
- }
- const environmentRealizeEndedAtMs = Date.now();
- activeEnvironmentLease = {
- ...activeEnvironmentLease,
- lease: realizationResult.lease,
- };
- persistedExecutionWorkspace =
- realizationResult.persistedExecutionWorkspace;
- // A sandbox realization may materialize or replace the durable workspace
- // after the host-side provisioning boundary above. Bind that final ID to
- // the issue before dispatch so warm turns reuse the exact same workspace
- // and lease scope instead of silently creating a per-run replacement.
- await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
- const workspaceRealization = realizationResult.workspaceRealization;
- const executionTarget = realizationResult.executionTarget;
- // Preserve the host-owned source before adapter context can share lease
- // metadata. A later copy-back failure must not adopt a rebound source.
- const workspaceRestoreSource = structuredClone(realizationResult.lease);
let instructionCopy: Awaited<ReturnType<typeof instructionCopies.prepare>> = null;
let instructionSave: Record<string, unknown> | null = null;
const instructionPreparationKey = createHash("sha256").update(JSON.stringify({
diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts
new file mode 100644
index 0000000000..c251f4e02a
--- /dev/null
+++ b/server/src/services/heartbeat/workspace-preparation.test.ts
@@ -0,0 +1,236 @@
+import { execFile as execFileCallback } from "node:child_process";
+import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, heartbeatRuns, issues, projects, projectWorkspaces, type Db } from "@paperclipai/db";
+import { eq, sql } from "drizzle-orm";
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
+import { environmentRunOrchestrator } from "../environment-run-orchestrator.js";
+import { environmentService } from "../environments.js";
+import { executionWorkspaceService } from "../execution-workspaces.js";
+import { instanceSettingsService } from "../instance-settings.js";
+import { issueService } from "../issues.js";
+import { workspaceOperationService } from "../workspace-operations.js";
+import { createHeartbeatWorkspaceResolver, resolveExecutionWorkspaceReuseRequestForIssue } from "./workspaces.js";
+import { prepareHeartbeatWorkspace, type HeartbeatWorkspacePreparationInput } from "./workspace-preparation.js";
+
+const execFile = promisify(execFileCallback);
+const support = await getEmbeddedPostgresTestSupport();
+
+describe.skipIf(!support.supported)("heartbeat workspace preparation boundary", () => {
+ let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+ let db: Db;
+ let home: string;
+
+ beforeAll(async () => {
+ home = await mkdtemp(join(tmpdir(), "paperclip-workspace-preparation-"));
+ vi.stubEnv("PAPERCLIP_HOME", home);
+ database = await startEmbeddedPostgresTestDatabase("paperclip-workspace-preparation");
+ db = createDb(database.connectionString);
+ await db.execute(sql`set client_min_messages = warning`);
+ }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS);
+ afterAll(async () => {
+ await db?.$client.end();
+ await database?.cleanup();
+ vi.unstubAllEnvs();
+ if (home) await rm(home, { recursive: true, force: true });
+ });
+ beforeEach(async () => {
+ vi.restoreAllMocks();
+ await db.execute(sql`truncate table companies restart identity cascade`);
+ await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
+ });
+
+ async function fixture(isolated = false) {
+ const [company] = await db.insert(companies).values({ name: "Workspace preparation", issuePrefix: "WSP" }).returning();
+ const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Workspace agent", status: "running", adapterType: "process" }).returning();
+ const [project] = await db.insert(projects).values({ companyId: company.id, name: "Source project" }).returning();
+ const source = join(home, project.id);
+ await mkdir(source);
+ await execFile("git", ["init", "--initial-branch=main", source]);
+ await execFile("git", ["-C", source, "config", "user.email", "[email protected]"]);
+ await execFile("git", ["-C", source, "config", "user.name", "Workspace test"]);
+ await writeFile(join(source, "README.md"), "source content\n");
+ await execFile("git", ["-C", source, "add", "."]);
+ await execFile("git", ["-C", source, "commit", "-m", "Source"]);
+ const [projectWorkspace] = await db.insert(projectWorkspaces).values({ companyId: company.id, projectId: project.id, name: "Source", cwd: source, isPrimary: true }).returning();
+ const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Prepare this workspace", identifier: "WSP-1", projectId: project.id, projectWorkspaceId: projectWorkspace.id, assigneeAgentId: agent.id }).returning();
+ const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();
+ const localEnvironment = await environmentService(db).ensureLocalEnvironment(company.id);
+ const envOrchestrator = environmentRunOrchestrator(db);
+ const resolver = createHeartbeatWorkspaceResolver(db);
+ const input: HeartbeatWorkspacePreparationInput = {
+ run, agent,
+ task: { issueRef: issue, issueId: issue.id, context: { issueId: issue.id, projectId: project.id }, executionProjectId: project.id, responsibleUserId: null, previousSessionParams: null },
+ policy: {
+ trustPreset: { kind: "standard", preset: "standard", boundary: null, sourcePresets: {} },
+ isolatedWorkspacesEnabled: true,
+ effectiveExecutionWorkspaceMode: isolated ? "isolated_workspace" : "shared_workspace",
+ requestedExecutionWorkspaceMode: isolated ? "isolated_workspace" : "shared_workspace",
+ useIsolatedTaskDirectory: false, nativeChatWorkspaceScope: null,
+ projectExecutionWorkspacePolicy: null, issueExecutionWorkspaceSettings: null, environmentExecutionWorkspaceSettings: null,
+ executionPolicy: { executionMode: "any" },
+ },
+ environment: { selectedEnvironmentId: localEnvironment.id, localEnvironment, selectedEnvironmentForConfig: localEnvironment, environmentResolution: { environmentId: localEnvironment.id, source: "default" }, resolvedInstanceSettings: await instanceSettingsService(db).get() },
+ config: { mergedConfig: isolated ? { workspaceStrategy: { type: "git_worktree", baseRef: "main", branchTemplate: "test-{{issue.identifier}}" } } : {}, configSnapshot: null, secretManifest: [] },
+ reuse: { requestedShouldReuseExisting: false, existingExecutionWorkspace: null, reusableExistingExecutionWorkspace: null, workspaceReuseRequest: resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: null, issueExecutionWorkspacePreference: null, existingExecutionWorkspaceStatus: null }), nativeRecoveryExecutionWorkspaceId: null, persistedNativeExecutionInput: null, isDotRun: false, runOptions: {} },
+ services: { envOrchestrator, executionWorkspacesSvc: executionWorkspaceService(db), workspaceOperationsSvc: workspaceOperationService(db), issuesSvc: issueService(db), ...resolver, appendRunEvent: vi.fn() },
+ controllerLease: { assertOwned: vi.fn(async () => {}) }, nativeRunnerPreparationSpans: [],
+ };
+ return { input, source, issue, run, envOrchestrator };
+ }
+ const readIssue = (id: string) => db.select().from(issues).where(eq(issues.id, id)).then(rows => rows[0]!);
+ const readRun = (id: string) => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)).then(rows => rows[0]!);
+ const leases = (id: string) => db.select().from(environmentLeases).where(eq(environmentLeases.heartbeatRunId, id));
+
+ async function reuse(input: HeartbeatWorkspacePreparationInput, workspace: NonNullable<Awaited<ReturnType<typeof prepareHeartbeatWorkspace>>["persistedExecutionWorkspace"]>) {
+ input.reuse.requestedShouldReuseExisting = true;
+ input.reuse.existingExecutionWorkspace = workspace;
+ input.reuse.reusableExistingExecutionWorkspace = workspace;
+ input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ issueExecutionWorkspaceId: workspace.id, issueExecutionWorkspacePreference: "reuse_existing", existingExecutionWorkspaceStatus: workspace.status });
+ input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: workspace.id, executionWorkspacePreference: "reuse_existing" };
+ const [run] = await db.insert(heartbeatRuns).values({ companyId: input.run.companyId, agentId: input.agent.id, issueId: input.task.issueId, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning();
+ input.run = run;
+ }
+
+ it("persists the workspace and issue binding before acquiring its environment", async () => {
+ const { input, source, issue, envOrchestrator } = await fixture();
+ const acquire = envOrchestrator.acquireForRun;
+ vi.spyOn(envOrchestrator, "acquireForRun").mockImplementation(async args => {
+ expect((await readIssue(issue.id)).executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);
+ expect((await readRun(input.run.id)).contextSnapshot?.executionWorkspaceId).toBe(args.persistedExecutionWorkspace?.id);
+ return acquire(args);
+ });
+ const result = await prepareHeartbeatWorkspace(db, input);
+ expect(result.executionWorkspace.cwd).toBe(source);
+ expect(result.persistedExecutionWorkspace).toMatchObject({ companyId: issue.companyId, projectId: issue.projectId, projectWorkspaceId: issue.projectWorkspaceId, mode: "shared_workspace" });
+ expect(await leases(input.run.id)).toMatchObject([{ status: "active", executionWorkspaceId: result.persistedExecutionWorkspace!.id }]);
+ expect(input.nativeRunnerPreparationSpans.map(span => span.name)).toEqual(["environment.acquire", "environment.workspace.realize"]);
+ expect(input.controllerLease.assertOwned).toHaveBeenCalledTimes(2);
+ });
+
+ it("creates a real isolated worktree and reuses its recorded workspace", async () => {
+ const { input, source, issue } = await fixture(true);
+ const first = await prepareHeartbeatWorkspace(db, input);
+ expect(first.executionWorkspace.cwd).not.toBe(source);
+ expect(await readFile(join(first.executionWorkspace.cwd, "README.md"), "utf8")).toBe("source content\n");
+ expect(await readIssue(issue.id)).toMatchObject({ executionWorkspaceId: first.persistedExecutionWorkspace!.id, executionWorkspacePreference: "reuse_existing" });
+ await writeFile(join(first.executionWorkspace.cwd, "retained.txt"), "keep this work");
+ await reuse(input, first.persistedExecutionWorkspace!);
+ const next = await prepareHeartbeatWorkspace(db, input);
+ expect(next.reusedExecutionWorkspace).toMatchObject({ cwd: first.executionWorkspace.cwd });
+ expect(next.persistedExecutionWorkspace!.id).toBe(first.persistedExecutionWorkspace!.id);
+ expect(await readFile(join(next.executionWorkspace.cwd, "retained.txt"), "utf8")).toBe("keep this work");
+ });
+
+ it("keeps the admitted recovery workspace when the issue now points elsewhere", async () => {
+ const { input, issue } = await fixture(true);
+ const first = await prepareHeartbeatWorkspace(db, input);
+ await reuse(input, first.persistedExecutionWorkspace!);
+ await db.update(issues).set({ executionWorkspaceId: null, projectWorkspaceId: null }).where(eq(issues.id, issue.id));
+ input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: null, projectWorkspaceId: null };
+ input.reuse.nativeRecoveryExecutionWorkspaceId = first.persistedExecutionWorkspace!.id;
+ const anchor = vi.spyOn(input.services, "resolveReusedGitWorkspaceAnchor");
+ const next = await prepareHeartbeatWorkspace(db, input);
+ expect(next.persistedExecutionWorkspace!.id).toBe(first.persistedExecutionWorkspace!.id);
+ expect(anchor).toHaveBeenCalledWith(expect.objectContaining({ immutableNativeBinding: true, explicitProjectWorkspaceId: first.persistedExecutionWorkspace!.projectWorkspaceId }));
+ expect(await readIssue(issue.id)).toMatchObject({ executionWorkspaceId: null, projectWorkspaceId: null });
+ });
+
+ it("removes a newly created worktree when persistence fails and preserves the original error", async () => {
+ const { input, source } = await fixture(true);
+ const error = new Error("workspace persistence unavailable");
+ let createdCwd = "";
+ vi.spyOn(input.services.executionWorkspacesSvc, "create").mockImplementation(async args => { createdCwd = args.cwd!; throw error; });
+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error);
+ expect(createdCwd).not.toBe("");
+ await expect(access(createdCwd)).rejects.toThrow();
+ expect(await readFile(join(source, "README.md"), "utf8")).toBe("source content\n");
+ expect(await leases(input.run.id)).toHaveLength(0);
+ });
+
+ it("preserves a reused worktree when its metadata update fails", async () => {
+ const { input } = await fixture(true);
+ const first = await prepareHeartbeatWorkspace(db, input);
+ await reuse(input, first.persistedExecutionWorkspace!);
+ await writeFile(join(first.executionWorkspace.cwd, "retained.txt"), "unsaved work");
+ const error = new Error("workspace update unavailable");
+ vi.spyOn(input.services.executionWorkspacesSvc, "update").mockRejectedValueOnce(error);
+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error);
+ expect(await readFile(join(first.executionWorkspace.cwd, "retained.txt"), "utf8")).toBe("unsaved work");
+ expect(await leases(input.run.id)).toHaveLength(0);
+ });
+
+ it.each(["acquire", "realize", "ownership"] as const)("preserves %s errors and durable lease state for outer cleanup", async failure => {
+ const { input, envOrchestrator } = await fixture();
+ const error = new Error(`${failure} failed`);
+ if (failure === "acquire") vi.spyOn(envOrchestrator, "acquireForRun").mockRejectedValueOnce(error);
+ if (failure === "realize") vi.spyOn(envOrchestrator, "realizeForRun").mockRejectedValueOnce(error);
+ if (failure === "ownership") vi.mocked(input.controllerLease.assertOwned).mockResolvedValueOnce(undefined).mockRejectedValueOnce(error);
+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toBe(error);
+ expect(input.nativeRunnerPreparationSpans.at(-1)).toMatchObject({ name: failure === "realize" ? "environment.workspace.realize" : "environment.acquire", outcome: "failed" });
+ expect((await readRun(input.run.id)).status).toBe("running");
+ expect(await leases(input.run.id)).toHaveLength(failure === "acquire" ? 0 : 1);
+ if (failure !== "acquire") {
+ expect(await leases(input.run.id)).toMatchObject([{ status: "active" }]);
+ // The production executor terminalizes first, then releases by durable run ID.
+ await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, input.run.id));
+ await envOrchestrator.releaseForRun({ heartbeatRunId: input.run.id, companyId: input.run.companyId, agentId: input.agent.id, status: "released" });
+ expect(await leases(input.run.id)).toMatchObject([{ status: "released" }]);
+ }
+ });
+
+ it("checks the acquired environment allowlist before realization", async () => {
+ const { input, envOrchestrator } = await fixture();
+ input.policy.executionPolicy = { managedSandboxOnly: true };
+ const realize = vi.spyOn(envOrchestrator, "realizeForRun");
+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("forbids local execution");
+ expect(realize).not.toHaveBeenCalled();
+ expect(await leases(input.run.id)).toMatchObject([{ status: "active" }]);
+ });
+
+ it("rejects low-trust local execution before resolving or provisioning a workspace", async () => {
+ const { input } = await fixture(true);
+ input.policy.trustPreset = {
+ kind: "low_trust_review", preset: "low_trust_review", sourcePresets: {},
+ boundary: { mode: "low_trust_review", companyId: input.agent.companyId, rootIssueId: input.task.issueId! },
+ };
+ const resolve = vi.spyOn(input.services, "resolveWorkspaceForRun");
+ const create = vi.spyOn(input.services.executionWorkspacesSvc, "create");
+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("requires a sandbox environment");
+ expect(resolve).not.toHaveBeenCalled();
+ expect(create).not.toHaveBeenCalled();
+ expect(await leases(input.run.id)).toHaveLength(0);
+ });
+
+ it("rejects remote recovery without its retained sync binding before acquiring a lease", async () => {
+ const { input, envOrchestrator } = await fixture();
+ input.reuse.runOptions.nativeRestartRecovery = {
+ kind: "reattach_remote_runner", runId: input.run.id, leaseOwner: "controller",
+ controllerGeneration: 1, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null,
+ remote: { providerLeaseId: "original-provider-lease", remoteCwd: "/original" },
+ };
+ const acquire = vi.spyOn(envOrchestrator, "acquireForRun");
+ await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("native_remote_recovery_lease_mismatch");
+ expect(acquire).not.toHaveBeenCalled();
+ expect(await leases(input.run.id)).toHaveLength(0);
+ expect(input.nativeRunnerPreparationSpans).toMatchObject([{ name: "environment.acquire", outcome: "failed" }]);
+ });
+
+ it("binds a replacement workspace from realization and snapshots the restore lease", async () => {
+ const { input, envOrchestrator, issue } = await fixture();
+ const realize = envOrchestrator.realizeForRun;
+ vi.spyOn(envOrchestrator, "realizeForRun").mockImplementation(async args => {
+ const result = await realize(args);
+ const workspace = await input.services.executionWorkspacesSvc.create({ companyId: issue.companyId, projectId: issue.projectId!, projectWorkspaceId: issue.projectWorkspaceId, sourceIssueId: issue.id, name: "Realized workspace", mode: "shared_workspace", strategyType: "project_primary", cwd: args.executionWorkspace.cwd });
+ return { ...result, persistedExecutionWorkspace: workspace, lease: { ...result.lease, metadata: { remoteCwd: "/original" } } };
+ });
+ const result = await prepareHeartbeatWorkspace(db, input);
+ expect((await readIssue(issue.id)).executionWorkspaceId).toBe(result.persistedExecutionWorkspace!.id);
+ expect(result.persistedExecutionWorkspace!.name).toBe("Realized workspace");
+ result.activeEnvironmentLease.lease.metadata!.remoteCwd = "/rebound";
+ expect(result.workspaceRestoreSource.metadata?.remoteCwd).toBe("/original");
+ });
+});
diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts
new file mode 100644
index 0000000000..cb699b9482
--- /dev/null
+++ b/server/src/services/heartbeat/workspace-preparation.ts
@@ -0,0 +1,1072 @@
+import {
+ resolveWorkspaceAfterLowTrustPreflight,
+ WorkspaceValidationFailure,
+ stripHostWorkspaceProvisionForLowTrustSandbox,
+ assertGitWorktreeBaseWorkspaceReady,
+ buildEffectiveRunWorkspaceConfigMetadata,
+ resolveExecutionWorkspaceConfigFreshness,
+ resolveExecutionWorkspaceReuseProvisioningPolicy,
+ provisionExecutionWorkspaceForFreshnessDecision,
+ mergeExecutionWorkspaceMetadataForPersistence,
+ resolveExecutionWorkspaceBranchOwnership,
+ reconcileReusedExecutionWorkspaceProjectWorkspaceId,
+ recordWorkspaceConfigFreshnessOperation,
+ prepareProjectRepositoryWorkspaces,
+} from "./workspaces.js";
+import { resolvePaperclipInstanceRoot } from "../../home-paths.js";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { and, asc, eq } from "drizzle-orm";
+import { type ExecutionWorkspace } from "@paperclipai/shared";
+import { heartbeatRuns, projectWorkspaces } from "@paperclipai/db";
+import { getStartupTracer } from "../../instrumentation.js";
+import { createHostDuplexObservabilityRecorder } from "../duplex-observability-recorder.js";
+import { incrementToolRuntimeMetricCounter } from "../tool-runtime-metrics.js";
+import { logger } from "../../middleware/logger.js";
+import { createGitRemoteAuthProvider } from "../git-credentials.js";
+import { readNativeWorkspaceSyncReference } from "../native-runtime/index.js";
+import { parseObject } from "../../adapters/utils.js";
+import { materializeNativeChatTaskRoot } from "../native-runtime/native-chat-workspace.js";
+import { materializeIsolatedTaskDirectory } from "../isolated-task-directory.js";
+import {
+ cleanupExecutionWorkspaceArtifacts,
+ ensurePersistedExecutionWorkspaceAvailable,
+ realizeExecutionWorkspace,
+ type ExecutionWorkspaceInput,
+ type RealizedExecutionWorkspace,
+} from "../workspace-runtime.js";
+import {
+ readManagedWorktreeInstanceOwnership,
+ WORKTREE_INSTANCE_ROOT_METADATA_KEY,
+} from "../workspace-instance-cleanup.js";
+import { isRuntimeOwnedGitBranch } from "../execution-workspace-branch-ownership.js";
+import {
+ issueExecutionWorkspaceModeForPersistedWorkspace,
+ resolveEffectiveWorkspaceStrategyType,
+} from "../execution-workspace-policy.js";
+import { evaluateExecutionAllowlist } from "../execution-allowlist.js";
+import type { Db, agents } from "@paperclipai/db";
+import type {
+ Environment,
+ IssueExecutionWorkspaceSettings,
+ ProjectExecutionWorkspacePolicy,
+} from "@paperclipai/shared";
+import type { NativeExecutionInput } from "../../vendor/paperclip-runner/index.js";
+import type { NativeRunHistoricalSpan } from "../native-runtime/native-run-trace.js";
+import type { NativeRestartRecoveryClaim } from "../native-runtime/index.js";
+import type { NativeChatWorkspaceScope } from "../native-runtime/native-chat-workspace.js";
+import type { TrustPresetResolution } from "../trust-preset-resolver.js";
+import type { environmentRunOrchestrator } from "../environment-run-orchestrator.js";
+import type { instanceSettingsService } from "../instance-settings.js";
+import type { issueService } from "../issues.js";
+import type { executionWorkspaceService } from "../execution-workspaces.js";
+import type { workspaceOperationService } from "../workspace-operations.js";
+import type { watchLegacyControllerLease } from "../legacy-controller-lease.js";
+import type {
+ createHeartbeatWorkspaceResolver,
+ resolveExecutionWorkspaceReuseRequestForIssue,
+} from "./workspaces.js";
+import type { createHeartbeatLifecycle } from "./run-lifecycle.js";
+import type {
+ ParsedExecutionWorkspaceMode,
+ ExecutionWorkspaceEnvironmentResolution,
+} from "../execution-workspace-policy.js";
+
+type WorkspaceResolver = ReturnType<typeof createHeartbeatWorkspaceResolver>;
+type WorkspaceMetadataInput = Parameters<typeof buildEffectiveRunWorkspaceConfigMetadata>[0];
+
+/** Prepares the workspace and environment; dispatch and terminal lease release stay with the caller. */
+export interface HeartbeatWorkspacePreparationInput {
+ run: typeof heartbeatRuns.$inferSelect;
+ agent: typeof agents.$inferSelect;
+ task: {
+ issueRef: (NonNullable<Parameters<typeof realizeExecutionWorkspace>[0]["issue"]> & {
+ projectId: string | null;
+ projectWorkspaceId: string | null;
+ executionWorkspaceId: string | null;
+ executionWorkspacePreference: string | null;
+ }) | null;
+ issueId: string | null;
+ context: Record<string, unknown>;
+ executionProjectId: string | null;
+ responsibleUserId: string | null;
+ previousSessionParams: Record<string, unknown> | null;
+ };
+ policy: {
+ trustPreset: TrustPresetResolution;
+ isolatedWorkspacesEnabled: boolean;
+ effectiveExecutionWorkspaceMode: ParsedExecutionWorkspaceMode;
+ requestedExecutionWorkspaceMode: ParsedExecutionWorkspaceMode;
+ useIsolatedTaskDirectory: boolean;
+ nativeChatWorkspaceScope: NativeChatWorkspaceScope | null;
+ projectExecutionWorkspacePolicy: ProjectExecutionWorkspacePolicy | null;
+ issueExecutionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;
+ environmentExecutionWorkspaceSettings: IssueExecutionWorkspaceSettings | null;
+ executionPolicy: Parameters<typeof evaluateExecutionAllowlist>[0];
+ };
+ environment: {
+ selectedEnvironmentId: string;
+ localEnvironment: Environment;
+ selectedEnvironmentForConfig: Environment | null;
+ environmentResolution: ExecutionWorkspaceEnvironmentResolution;
+ resolvedInstanceSettings: Awaited<ReturnType<ReturnType<typeof instanceSettingsService>["get"]>>;
+ };
+ config: {
+ mergedConfig: Record<string, unknown>;
+ configSnapshot: WorkspaceMetadataInput["configSnapshot"];
+ secretManifest: WorkspaceMetadataInput["secretManifest"];
+ };
+ reuse: {
+ requestedShouldReuseExisting: boolean;
+ existingExecutionWorkspace: ExecutionWorkspace | null;
+ reusableExistingExecutionWorkspace: ExecutionWorkspace | null;
+ workspaceReuseRequest: ReturnType<typeof resolveExecutionWorkspaceReuseRequestForIssue>;
+ nativeRecoveryExecutionWorkspaceId: string | null;
+ persistedNativeExecutionInput: NativeExecutionInput | null;
+ isDotRun: boolean;
+ runOptions: { nativeRestartRecovery?: NativeRestartRecoveryClaim };
+ };
+ services: {
+ envOrchestrator: Pick<ReturnType<typeof environmentRunOrchestrator>, "resolveEnvironment" | "acquireForRun" | "realizeForRun">;
+ executionWorkspacesSvc: Pick<ReturnType<typeof executionWorkspaceService>, "create" | "update">;
+ workspaceOperationsSvc: Pick<ReturnType<typeof workspaceOperationService>, "createRecorder">;
+ issuesSvc: Pick<ReturnType<typeof issueService>, "update">;
+ resolveWorkspaceForRun: WorkspaceResolver["resolveWorkspaceForRun"];
+ resolveReusedGitWorkspaceAnchor: WorkspaceResolver["resolveReusedGitWorkspaceAnchor"];
+ appendRunEvent: ReturnType<typeof createHeartbeatLifecycle>["appendRunEvent"];
+ };
+ controllerLease: Pick<ReturnType<typeof watchLegacyControllerLease>, "assertOwned">;
+ nativeRunnerPreparationSpans: NativeRunHistoricalSpan[];
+}
+
+function readNonEmptyString(value: unknown): string | null {
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
+}
+
+export async function prepareHeartbeatWorkspace(db: Db, input: HeartbeatWorkspacePreparationInput) {
+ const { run, agent, controllerLease, nativeRunnerPreparationSpans } = input;
+ const {
+ issueRef,
+ issueId,
+ context,
+ executionProjectId,
+ responsibleUserId,
+ previousSessionParams,
+ } = input.task;
+ const {
+ trustPreset,
+ isolatedWorkspacesEnabled,
+ effectiveExecutionWorkspaceMode,
+ requestedExecutionWorkspaceMode,
+ useIsolatedTaskDirectory,
+ nativeChatWorkspaceScope,
+ projectExecutionWorkspacePolicy,
+ issueExecutionWorkspaceSettings,
+ environmentExecutionWorkspaceSettings,
+ executionPolicy,
+ } = input.policy;
+ const {
+ selectedEnvironmentId,
+ localEnvironment,
+ selectedEnvironmentForConfig,
+ environmentResolution,
+ resolvedInstanceSettings,
+ } = input.environment;
+ const { mergedConfig, configSnapshot, secretManifest } = input.config;
+ const {
+ requestedShouldReuseExisting,
+ existingExecutionWorkspace,
+ reusableExistingExecutionWorkspace,
+ workspaceReuseRequest,
+ nativeRecoveryExecutionWorkspaceId,
+ persistedNativeExecutionInput,
+ isDotRun,
+ runOptions,
+ } = input.reuse;
+ const {
+ envOrchestrator,
+ executionWorkspacesSvc,
+ workspaceOperationsSvc,
+ issuesSvc,
+ resolveWorkspaceForRun,
+ resolveReusedGitWorkspaceAnchor,
+ appendRunEvent,
+ } = input.services;
+ const {
+ selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,
+ workspace: resolvedWorkspace,
+ } = await resolveWorkspaceAfterLowTrustPreflight({
+ db,
+ trustPreset,
+ isolatedWorkspacesEnabled,
+ effectiveExecutionWorkspaceMode,
+ issue: issueRef
+ ? {
+ companyId: agent.companyId,
+ id: issueRef.id,
+ projectId: issueRef.projectId,
+ }
+ : null,
+ resolveSelectedEnvironmentDriver: async () => {
+ const preflightEnvironment = await envOrchestrator.resolveEnvironment(
+ {
+ companyId: agent.companyId,
+ selectedEnvironmentId,
+ localEnvironmentId: localEnvironment.id,
+ },
+ );
+ return preflightEnvironment.driver;
+ },
+ resolveWorkspace: async () => {
+ if (isDotRun) {
+ // This is private controller storage, never a provider filesystem.
+ // v6 projects workspace.access=none and cwd=null to Dot.
+ const cwd = path.resolve(resolvePaperclipInstanceRoot(), "runtime", "paperclip-runner", "dot-controllers", agent.companyId, run.id);
+ await fs.mkdir(cwd, { recursive: true, mode: 0o700 });
+ return { cwd, source: "agent_home" as const, projectId: null, workspaceId: null, repoUrl: null, repoRef: null,
+ workspaceHints: [], warnings: [], baseCwdFallback: false, materializationFailures: [], additionalWorkspaces: [], referencedProjectFailures: [] };
+ }
+ if (useIsolatedTaskDirectory && issueRef) {
+ const cwd = await materializeIsolatedTaskDirectory({
+ companyId: agent.companyId,
+ issueId: issueRef.id,
+ });
+ if (reusableExistingExecutionWorkspace && (
+ reusableExistingExecutionWorkspace.companyId !== agent.companyId ||
+ reusableExistingExecutionWorkspace.projectId !== issueRef.projectId ||
+ reusableExistingExecutionWorkspace.sourceIssueId !== issueRef.id ||
+ reusableExistingExecutionWorkspace.mode !== "isolated_workspace" ||
+ reusableExistingExecutionWorkspace.strategyType !== "project_primary" ||
+ reusableExistingExecutionWorkspace.cwd !== cwd
+ )) {
+ throw new WorkspaceValidationFailure("The existing execution workspace is not this task's isolated directory.", {
+ workspaceValidation: { reason: "isolated_task_directory_binding_mismatch", issueId: issueRef.id },
+ });
+ }
+ return resolveWorkspaceForRun(agent, context, previousSessionParams, {
+ executionEnvironmentDriver: selectedEnvironmentForConfig?.driver ?? null,
+ anchorWorkspace: {
+ cwd,
+ source: "task_session",
+ projectId: issueRef.projectId,
+ workspaceId: null,
+ repoUrl: null,
+ repoRef: null,
+ workspaceHints: [],
+ warnings: [],
+ baseCwdFallback: false,
+ materializationFailures: [],
+ },
+ });
+ }
+ if (nativeChatWorkspaceScope && !nativeChatWorkspaceScope.projectId) {
+ const cwd = await materializeNativeChatTaskRoot(
+ nativeChatWorkspaceScope,
+ );
+ return {
+ cwd,
+ source: "task_session" as const,
+ projectId: null,
+ workspaceId: null,
+ repoUrl: null,
+ repoRef: null,
+ workspaceHints: [],
+ warnings: [],
+ baseCwdFallback: false,
+ materializationFailures: [],
+ additionalWorkspaces: [],
+ referencedProjectFailures: [],
+ };
+ }
+ const workspace = await resolveWorkspaceForRun(
+ agent,
+ context,
+ previousSessionParams,
+ {
+ useProjectWorkspace:
+ requestedExecutionWorkspaceMode !== "agent_default",
+ anchorWorkspace: requestedShouldReuseExisting && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"
+ ? await resolveReusedGitWorkspaceAnchor({
+ agent,
+ workspace: reusableExistingExecutionWorkspace,
+ responsibleUserId,
+ immutableNativeBinding: Boolean(nativeRecoveryExecutionWorkspaceId),
+ projectId: nativeRecoveryExecutionWorkspaceId
+ ? reusableExistingExecutionWorkspace.projectId
+ : issueRef?.projectId ?? readNonEmptyString(context.projectId),
+ explicitProjectWorkspaceId: nativeRecoveryExecutionWorkspaceId
+ ? reusableExistingExecutionWorkspace.projectWorkspaceId
+ : readNonEmptyString(context.projectWorkspaceId),
+ issueId,
+ runId: run.id,
+ })
+ : undefined,
+ // Thread the selected environment driver so run-workspace resolution can tell a local
+ // target from a remote one, and a confined sandbox target from an unconfined remote
+ // target. A remote run resolves referenced projects only for the confined sandbox
+ // transport with the remote flag on. This never changes the anchor workspace.
+ executionEnvironmentDriver:
+ selectedEnvironmentForConfig?.driver ?? null,
+ },
+ );
+ // Additional referenced projects are a separate trusted Board
+ // capability, not extra readable roots for an external conversation.
+ return nativeChatWorkspaceScope
+ ? {
+ ...workspace,
+ additionalWorkspaces: [],
+ referencedProjectFailures: [],
+ }
+ : workspace;
+ },
+ });
+ const hostExecutionWorkspaceConfig = isDotRun ? {} :
+ stripHostWorkspaceProvisionForLowTrustSandbox({
+ config: mergedConfig,
+ trustPreset,
+ selectedEnvironmentDriver: lowTrustPreflightEnvironmentDriver,
+ });
+ const executionWorkspaceBase = {
+ baseCwd: resolvedWorkspace.cwd,
+ source: resolvedWorkspace.source,
+ projectId: resolvedWorkspace.projectId,
+ workspaceId: resolvedWorkspace.workspaceId,
+ repoUrl: resolvedWorkspace.repoUrl,
+ repoRef: resolvedWorkspace.repoRef,
+ additionalWorkspaces: resolvedWorkspace.additionalWorkspaces,
+ } satisfies ExecutionWorkspaceInput;
+ await assertGitWorktreeBaseWorkspaceReady({
+ requestedExecutionWorkspaceMode,
+ config: hostExecutionWorkspaceConfig,
+ issue: issueRef,
+ base: executionWorkspaceBase,
+ anchor: {
+ baseCwdFallback: resolvedWorkspace.baseCwdFallback,
+ materializationFailures: resolvedWorkspace.materializationFailures,
+ localPathOnlyWorkspace: resolvedWorkspace.localPathOnlyWorkspace,
+ },
+ });
+ const workspaceStrategyForFingerprint = parseObject(
+ hostExecutionWorkspaceConfig.workspaceStrategy,
+ );
+ const workspaceStrategyFingerprintValue =
+ Object.keys(workspaceStrategyForFingerprint).length > 0
+ ? workspaceStrategyForFingerprint
+ : null;
+ const latestWorkspaceStrategyType = resolveEffectiveWorkspaceStrategyType(
+ requestedExecutionWorkspaceMode,
+ hostExecutionWorkspaceConfig,
+ );
+ const selectedEnvironmentConfigForFingerprint = parseObject(
+ selectedEnvironmentForConfig?.config,
+ );
+ const workspaceEnvironmentFingerprint = selectedEnvironmentForConfig
+ ? {
+ selectionSource: environmentResolution.source,
+ selectedEnvironmentId,
+ driver: selectedEnvironmentForConfig.driver,
+ provider: readNonEmptyString(
+ selectedEnvironmentConfigForFingerprint.provider,
+ ),
+ config: selectedEnvironmentForConfig.config,
+ configRevisionAt:
+ selectedEnvironmentForConfig.updatedAt instanceof Date
+ ? selectedEnvironmentForConfig.updatedAt.toISOString()
+ : (selectedEnvironmentForConfig.updatedAt ?? null),
+ executionPolicy,
+ }
+ : null;
+ const workspaceRealizationFingerprint = {
+ environmentDriver: selectedEnvironmentForConfig?.driver ?? null,
+ environmentProvider: readNonEmptyString(
+ selectedEnvironmentConfigForFingerprint.provider,
+ ),
+ trustPreset: trustPreset.kind,
+ lowTrustSandboxDriver: lowTrustPreflightEnvironmentDriver,
+ };
+ const workspaceFreshnessSource = resolvedWorkspace.freshnessSource ?? executionWorkspaceBase;
+ const latestWorkspaceConfigMetadata =
+ buildEffectiveRunWorkspaceConfigMetadata({
+ mode: requestedExecutionWorkspaceMode,
+ projectId: workspaceFreshnessSource.projectId,
+ projectWorkspaceId: workspaceFreshnessSource.workspaceId,
+ strategyType: latestWorkspaceStrategyType,
+ workspaceStrategy: workspaceStrategyFingerprintValue,
+ repoUrl: workspaceFreshnessSource.repoUrl,
+ repoRef:
+ readNonEmptyString(workspaceStrategyForFingerprint.baseRef) ??
+ workspaceFreshnessSource.repoRef,
+ configSnapshot,
+ environment: workspaceEnvironmentFingerprint,
+ realization: workspaceRealizationFingerprint,
+ secretManifest,
+ });
+ const inferredExistingWorkspaceConfigMetadata =
+ reusableExistingExecutionWorkspace
+ ? buildEffectiveRunWorkspaceConfigMetadata({
+ mode: issueExecutionWorkspaceModeForPersistedWorkspace(
+ reusableExistingExecutionWorkspace.mode,
+ ),
+ projectId: reusableExistingExecutionWorkspace.projectId,
+ projectWorkspaceId:
+ reusableExistingExecutionWorkspace.projectWorkspaceId,
+ strategyType: reusableExistingExecutionWorkspace.strategyType,
+ workspaceStrategy: workspaceStrategyFingerprintValue
+ ? {
+ ...workspaceStrategyFingerprintValue,
+ type: reusableExistingExecutionWorkspace.strategyType,
+ ...(reusableExistingExecutionWorkspace.baseRef
+ ? { baseRef: reusableExistingExecutionWorkspace.baseRef }
+ : {}),
+ }
+ : { type: reusableExistingExecutionWorkspace.strategyType },
+ repoUrl: reusableExistingExecutionWorkspace.repoUrl,
+ repoRef: reusableExistingExecutionWorkspace.baseRef,
+ configSnapshot: reusableExistingExecutionWorkspace.config,
+ environment: workspaceEnvironmentFingerprint,
+ realization: workspaceRealizationFingerprint,
+ secretManifest,
+ evaluatedAt: latestWorkspaceConfigMetadata.evaluatedAt,
+ })
+ : null;
+ const workspaceConfigFreshness = resolveExecutionWorkspaceConfigFreshness(
+ {
+ hasExistingWorkspace:
+ requestedShouldReuseExisting &&
+ Boolean(reusableExistingExecutionWorkspace),
+ existingWorkspaceMetadata:
+ reusableExistingExecutionWorkspace?.metadata ?? null,
+ inferredMetadata: inferredExistingWorkspaceConfigMetadata,
+ nextMetadata: latestWorkspaceConfigMetadata,
+ },
+ );
+ const workspaceReuseProvisioningPolicy =
+ resolveExecutionWorkspaceReuseProvisioningPolicy({
+ requestedShouldReuseExisting,
+ workspaceConfigFreshness,
+ });
+ const workspaceOperationRecorder = workspaceOperationsSvc.createRecorder({
+ companyId: agent.companyId,
+ heartbeatRunId: run.id,
+ executionWorkspaceId:
+ workspaceReuseProvisioningPolicy.shouldRestoreExistingWorkspace
+ ? workspaceReuseRequest.requestedExecutionWorkspaceId
+ : null,
+ issueId,
+ });
+ // The run-scoped provider resolves the active identity at each Git operation,
+ // including base-ref refreshes, workspace realization, and restore.
+ const workspaceGitAuthProvider = createGitRemoteAuthProvider(
+ db,
+ agent.companyId,
+ {
+ issueId,
+ heartbeatRunId: run.id,
+ responsibleUserId: run.responsibleUserId,
+ agentId: agent.id,
+ },
+ );
+ const {
+ executionWorkspace,
+ reusedExecutionWorkspace,
+ policy: resolvedWorkspaceReusePolicy,
+ } = isDotRun ? { executionWorkspace: { ...executionWorkspaceBase, strategy: "project_primary" as const, cwd: resolvedWorkspace.cwd, branchName: null, worktreePath: null, warnings: [], created: false, branchCreatedByRuntime: false } as RealizedExecutionWorkspace, reusedExecutionWorkspace: false, policy: workspaceReuseProvisioningPolicy } : await provisionExecutionWorkspaceForFreshnessDecision<RealizedExecutionWorkspace>(
+ {
+ requestedShouldReuseExisting,
+ existingExecutionWorkspaceId:
+ workspaceReuseRequest.requestedExecutionWorkspaceId,
+ issueRef,
+ runId: run.id,
+ workspaceConfigFreshness,
+ restoreExistingWorkspace: reusableExistingExecutionWorkspace
+ ? () =>
+ ensurePersistedExecutionWorkspaceAvailable({
+ db,
+ base: executionWorkspaceBase,
+ workspace: {
+ id: reusableExistingExecutionWorkspace.id,
+ mode: reusableExistingExecutionWorkspace.mode,
+ strategyType:
+ reusableExistingExecutionWorkspace.strategyType,
+ cwd: reusableExistingExecutionWorkspace.cwd,
+ providerRef: reusableExistingExecutionWorkspace.providerRef,
+ projectId: reusableExistingExecutionWorkspace.projectId,
+ projectWorkspaceId:
+ reusableExistingExecutionWorkspace.projectWorkspaceId,
+ repoUrl: reusableExistingExecutionWorkspace.repoUrl,
+ baseRef: reusableExistingExecutionWorkspace.baseRef,
+ branchName: reusableExistingExecutionWorkspace.branchName,
+ metadata:
+ reusableExistingExecutionWorkspace.metadata as Record<
+ string,
+ unknown
+ > | null,
+ config: {
+ provisionCommand:
+ configSnapshot?.provisionCommand ??
+ reusableExistingExecutionWorkspace.config
+ ?.provisionCommand ??
+ projectExecutionWorkspacePolicy?.workspaceStrategy
+ ?.provisionCommand ??
+ null,
+ runtimeProvisionCommand:
+ configSnapshot?.runtimeProvisionCommand ??
+ reusableExistingExecutionWorkspace.config
+ ?.runtimeProvisionCommand ??
+ projectExecutionWorkspacePolicy?.workspaceStrategy
+ ?.runtimeProvisionCommand ??
+ null,
+ },
+ },
+ issue: issueRef,
+ agent: {
+ id: agent.id,
+ name: agent.name,
+ companyId: agent.companyId,
+ },
+ heartbeatRunId: run.id,
+ enableWorkspaceBranchReconcileForward:
+ resolvedInstanceSettings.experimental
+ .enableWorkspaceBranchReconcileForward,
+ enableWorkspaceDirtyQuarantineRepair:
+ resolvedInstanceSettings.experimental
+ .enableWorkspaceDirtyQuarantineRepair,
+ recorder: workspaceOperationRecorder,
+ resolveGitAuth: workspaceGitAuthProvider,
+ })
+ : null,
+ realizeWorkspace: () =>
+ realizeExecutionWorkspace({
+ db,
+ base: executionWorkspaceBase,
+ config: hostExecutionWorkspaceConfig,
+ issue: issueRef,
+ agent: {
+ id: agent.id,
+ name: agent.name,
+ companyId: agent.companyId,
+ },
+ recordedBranchOwnership:
+ existingExecutionWorkspace?.status !== "archived" &&
+ existingExecutionWorkspace?.branchName
+ ? {
+ branchName: existingExecutionWorkspace.branchName,
+ createdByRuntime: isRuntimeOwnedGitBranch(
+ existingExecutionWorkspace.metadata,
+ ),
+ }
+ : null,
+ heartbeatRunId: run.id,
+ enableWorkspaceBranchReconcileForward:
+ resolvedInstanceSettings.experimental
+ .enableWorkspaceBranchReconcileForward,
+ enableWorkspaceDirtyQuarantineRepair:
+ resolvedInstanceSettings.experimental
+ .enableWorkspaceDirtyQuarantineRepair,
+ recorder: workspaceOperationRecorder,
+ resolveGitAuth: workspaceGitAuthProvider,
+ }),
+ },
+ );
+ const resolvedProjectId =
+ executionWorkspace.projectId ??
+ issueRef?.projectId ??
+ executionProjectId ??
+ null;
+ const resolvedProjectWorkspaceId =
+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace && reusableExistingExecutionWorkspace?.strategyType === "git_worktree"
+ ? reusableExistingExecutionWorkspace.projectWorkspaceId
+ : issueRef?.projectWorkspaceId ?? resolvedWorkspace.workspaceId ?? null;
+ let persistedExecutionWorkspace: ExecutionWorkspace | null = null;
+ let issueExecutionWorkspaceIdForRun =
+ issueRef?.executionWorkspaceId ?? null;
+ let issueProjectWorkspaceIdForRun = issueRef?.projectWorkspaceId ?? null;
+ let issueExecutionWorkspacePreferenceForRun =
+ issueRef?.executionWorkspacePreference ?? null;
+ let issueExecutionWorkspaceModeForRun =
+ issueExecutionWorkspaceSettings?.mode ?? null;
+ const warmReusableExecutionWorkspace =
+ selectedEnvironmentForConfig?.driver === "sandbox" &&
+ selectedEnvironmentConfigForFingerprint.reuseLease === true &&
+ selectedEnvironmentConfigForFingerprint.runnerLifecycleMode === "warm";
+ // Native provider checkpoints bind to the workspace row, including ordinary
+ // local shared workspaces. Persist that binding independently of the opt-in
+ // isolated-workspace UI, just as warm sandbox continuity already does.
+ const nativeSharedWorkspace = agent.adapterType === "paperclip_runner" &&
+ requestedExecutionWorkspaceMode === "shared_workspace";
+ const bindIssueToPersistedExecutionWorkspace = async (
+ workspace: ExecutionWorkspace | null,
+ ) => {
+ if (!issueId || !workspace || nativeRecoveryExecutionWorkspaceId) {
+ return;
+ }
+ const nextIssueWorkspaceMode =
+ issueExecutionWorkspaceModeForPersistedWorkspace(workspace.mode) ??
+ "agent_default";
+ const shouldSwitchIssueToExistingWorkspace =
+ issueRef?.executionWorkspacePreference === "reuse_existing" ||
+ requestedExecutionWorkspaceMode === "isolated_workspace" ||
+ requestedExecutionWorkspaceMode === "operator_branch" ||
+ warmReusableExecutionWorkspace || nativeSharedWorkspace;
+ const nextIssuePatch: Record<string, unknown> = {};
+ if (issueExecutionWorkspaceIdForRun !== workspace.id) {
+ nextIssuePatch.executionWorkspaceId = workspace.id;
+ }
+ if (
+ resolvedProjectWorkspaceId &&
+ issueProjectWorkspaceIdForRun !== resolvedProjectWorkspaceId
+ ) {
+ nextIssuePatch.projectWorkspaceId = resolvedProjectWorkspaceId;
+ }
+ if (
+ shouldSwitchIssueToExistingWorkspace &&
+ (issueExecutionWorkspacePreferenceForRun !== "reuse_existing" ||
+ issueExecutionWorkspaceModeForRun !== nextIssueWorkspaceMode)
+ ) {
+ nextIssuePatch.executionWorkspacePreference = "reuse_existing";
+ nextIssuePatch.executionWorkspaceSettings = {
+ ...(issueExecutionWorkspaceSettings ?? {}),
+ mode: nextIssueWorkspaceMode,
+ };
+ }
+ if (Object.keys(nextIssuePatch).length > 0) {
+ await issuesSvc.update(
+ issueId,
+ { ...nextIssuePatch, companyGuard: agent.companyId },
+ db,
+ undefined,
+ undefined,
+ { bindRuntimeSharedWorkspace: (warmReusableExecutionWorkspace || nativeSharedWorkspace) && workspace.mode === "shared_workspace" },
+ );
+ issueExecutionWorkspaceIdForRun = workspace.id;
+ issueProjectWorkspaceIdForRun =
+ resolvedProjectWorkspaceId ?? issueProjectWorkspaceIdForRun;
+ if (shouldSwitchIssueToExistingWorkspace) {
+ issueExecutionWorkspacePreferenceForRun = "reuse_existing";
+ issueExecutionWorkspaceModeForRun = nextIssueWorkspaceMode;
+ }
+ }
+ };
+ const baseExecutionWorkspaceMetadata =
+ mergeExecutionWorkspaceMetadataForPersistence({
+ existingMetadata:
+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace
+ ? (reusableExistingExecutionWorkspace?.metadata ?? null)
+ : null,
+ source: executionWorkspace.source,
+ // Attaching a new worktree to a pre-existing branch reports a fresh
+ // workspace, but must not make cleanup own the operator's branch.
+ createdByRuntime:
+ resolveExecutionWorkspaceBranchOwnership(executionWorkspace),
+ strategyType: executionWorkspace.strategy,
+ configSnapshot,
+ shouldReuseExisting:
+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace,
+ shouldRefreshConfigSnapshot:
+ resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
+ workspaceConfigMetadata:
+ resolvedWorkspaceReusePolicy.shouldPersistLatestWorkspaceConfigMetadata
+ ? latestWorkspaceConfigMetadata
+ : null,
+ baseRef: executionWorkspace.repoRef,
+ baseRefSha: executionWorkspace.baseRefSha ?? null,
+ });
+ let persistedWorktreeInstanceRoot =
+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&
+ typeof reusableExistingExecutionWorkspace?.metadata?.[
+ WORKTREE_INSTANCE_ROOT_METADATA_KEY
+ ] === "string"
+ ? reusableExistingExecutionWorkspace.metadata[
+ WORKTREE_INSTANCE_ROOT_METADATA_KEY
+ ]
+ : null;
+ if (
+ !persistedWorktreeInstanceRoot &&
+ executionWorkspace.strategy === "git_worktree" &&
+ executionWorkspace.worktreePath
+ ) {
+ try {
+ persistedWorktreeInstanceRoot =
+ (
+ await readManagedWorktreeInstanceOwnership(
+ executionWorkspace.worktreePath,
+ )
+ )?.instanceRoot ?? null;
+ } catch (error) {
+ logger.warn(
+ {
+ runId: run.id,
+ issueId,
+ executionWorkspaceCwd: executionWorkspace.cwd,
+ error: error instanceof Error ? error.message : String(error),
+ },
+ "Could not record managed worktree instance ownership",
+ );
+ }
+ }
+ const nextExecutionWorkspaceMetadata = {
+ ...baseExecutionWorkspaceMetadata,
+ ...(persistedWorktreeInstanceRoot
+ ? {
+ [WORKTREE_INSTANCE_ROOT_METADATA_KEY]:
+ persistedWorktreeInstanceRoot,
+ }
+ : {}),
+ };
+ const pendingForwardBranchReconcile =
+ executionWorkspace.pendingForwardBranchReconcile ?? null;
+ const branchNameForInitialPersistence =
+ pendingForwardBranchReconcile?.recordedBranchName ??
+ executionWorkspace.branchName;
+ try {
+ persistedExecutionWorkspace =
+ resolvedWorkspaceReusePolicy.shouldRestoreExistingWorkspace &&
+ reusableExistingExecutionWorkspace
+ ? await executionWorkspacesSvc.update(
+ reusableExistingExecutionWorkspace.id,
+ {
+ cwd: executionWorkspace.cwd,
+ repoUrl: executionWorkspace.repoUrl,
+ baseRef: executionWorkspace.repoRef,
+ branchName: branchNameForInitialPersistence,
+ providerType:
+ executionWorkspace.strategy === "git_worktree"
+ ? "git_worktree"
+ : "local_fs",
+ providerRef: executionWorkspace.worktreePath,
+ status: "active",
+ lastUsedAt: new Date(),
+ metadata: nextExecutionWorkspaceMetadata,
+ projectWorkspaceId:
+ reconcileReusedExecutionWorkspaceProjectWorkspaceId(
+ reusableExistingExecutionWorkspace.projectWorkspaceId,
+ resolvedProjectWorkspaceId,
+ ),
+ },
+ )
+ : resolvedProjectId
+ ? await executionWorkspacesSvc.create({
+ companyId: agent.companyId,
+ projectId: resolvedProjectId,
+ projectWorkspaceId: resolvedProjectWorkspaceId,
+ sourceIssueId: issueRef?.id ?? null,
+ mode:
+ requestedExecutionWorkspaceMode === "isolated_workspace"
+ ? "isolated_workspace"
+ : requestedExecutionWorkspaceMode === "operator_branch"
+ ? "operator_branch"
+ : requestedExecutionWorkspaceMode === "agent_default"
+ ? "adapter_managed"
+ : "shared_workspace",
+ strategyType:
+ executionWorkspace.strategy === "git_worktree"
+ ? "git_worktree"
+ : "project_primary",
+ name:
+ branchNameForInitialPersistence ??
+ issueRef?.identifier ??
+ `workspace-${agent.id.slice(0, 8)}`,
+ status: "active",
+ cwd: executionWorkspace.cwd,
+ repoUrl: executionWorkspace.repoUrl,
+ baseRef: executionWorkspace.repoRef,
+ branchName: branchNameForInitialPersistence,
+ providerType:
+ executionWorkspace.strategy === "git_worktree"
+ ? "git_worktree"
+ : "local_fs",
+ providerRef: executionWorkspace.worktreePath,
+ lastUsedAt: new Date(),
+ openedAt: new Date(),
+ metadata: nextExecutionWorkspaceMetadata,
+ })
+ : null;
+ } catch (error) {
+ if (executionWorkspace.created) {
+ try {
+ await cleanupExecutionWorkspaceArtifacts({
+ workspace: {
+ id:
+ reusableExistingExecutionWorkspace?.id ??
+ workspaceReuseRequest.requestedExecutionWorkspaceId ??
+ `transient-${run.id}`,
+ cwd: executionWorkspace.cwd,
+ providerType:
+ executionWorkspace.strategy === "git_worktree"
+ ? "git_worktree"
+ : "local_fs",
+ providerRef: executionWorkspace.worktreePath,
+ branchName: executionWorkspace.branchName,
+ repoUrl: executionWorkspace.repoUrl,
+ baseRef: executionWorkspace.repoRef,
+ projectId: resolvedProjectId,
+ projectWorkspaceId: resolvedProjectWorkspaceId,
+ sourceIssueId: issueRef?.id ?? null,
+ metadata: nextExecutionWorkspaceMetadata,
+ },
+ projectWorkspace: {
+ cwd: resolvedWorkspace.cwd,
+ cleanupCommand: null,
+ },
+ cleanupCommand: configSnapshot?.cleanupCommand ?? null,
+ teardownCommand:
+ configSnapshot?.teardownCommand ??
+ projectExecutionWorkspacePolicy?.workspaceStrategy
+ ?.teardownCommand ??
+ null,
+ recorder: workspaceOperationRecorder,
+ });
+ } catch (cleanupError) {
+ logger.warn(
+ {
+ runId: run.id,
+ issueId,
+ executionWorkspaceCwd: executionWorkspace.cwd,
+ cleanupError:
+ cleanupError instanceof Error
+ ? cleanupError.message
+ : String(cleanupError),
+ },
+ "Failed to cleanup realized execution workspace after persistence failure",
+ );
+ }
+ }
+ throw error;
+ }
+ await workspaceOperationRecorder.attachExecutionWorkspaceId(
+ persistedExecutionWorkspace?.id ?? null,
+ );
+ await recordWorkspaceConfigFreshnessOperation({
+ recorder: workspaceOperationRecorder,
+ runId: run.id,
+ decision: workspaceConfigFreshness,
+ hasExistingWorkspace: Boolean(reusableExistingExecutionWorkspace),
+ reuseRequested: requestedShouldReuseExisting,
+ workspaceReused: Boolean(reusedExecutionWorkspace),
+ configSnapshotRefreshed:
+ resolvedWorkspaceReusePolicy.shouldRefreshWorkspaceConfigSnapshot,
+ previousWorkspaceId:
+ workspaceReuseRequest.requestedExecutionWorkspaceId,
+ activeWorkspaceId: persistedExecutionWorkspace?.id ?? null,
+ });
+ if (
+ reusableExistingExecutionWorkspace &&
+ persistedExecutionWorkspace &&
+ reusableExistingExecutionWorkspace.id !==
+ persistedExecutionWorkspace.id &&
+ reusableExistingExecutionWorkspace.status === "active"
+ ) {
+ await executionWorkspacesSvc.update(
+ reusableExistingExecutionWorkspace.id,
+ {
+ status: "idle",
+ cleanupReason: null,
+ },
+ );
+ }
+ await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
+ const projectRepositoryPaths: string[] = [];
+ if (executionWorkspace.projectId && resolvedWorkspace.source === "project_primary" && !resolvedWorkspace.baseCwdFallback) {
+ const repositoryRows = await db.select().from(projectWorkspaces).where(and(
+ eq(projectWorkspaces.companyId, agent.companyId),
+ eq(projectWorkspaces.projectId, executionWorkspace.projectId),
+ )).orderBy(asc(projectWorkspaces.createdAt), asc(projectWorkspaces.id));
+ const repositories = await prepareProjectRepositoryWorkspaces({
+ cwd: executionWorkspace.cwd,
+ anchorRepoUrl: executionWorkspace.repoUrl,
+ workspaces: repositoryRows,
+ resolveGitAuth: workspaceGitAuthProvider,
+ });
+ const paths = new Map(repositories.map((repo) => [repo.workspaceId, repo.cwd]));
+ projectRepositoryPaths.push(...repositories.map((repo) => path.relative(executionWorkspace.cwd, repo.cwd)));
+ if (resolvedWorkspace.workspaceId) paths.set(resolvedWorkspace.workspaceId, executionWorkspace.cwd);
+ resolvedWorkspace.workspaceHints = resolvedWorkspace.workspaceHints.map((hint) => ({
+ ...hint, cwd: paths.get(hint.workspaceId) ?? hint.cwd,
+ }));
+ }
+ if (persistedExecutionWorkspace) {
+ context.executionWorkspaceId = persistedExecutionWorkspace.id;
+ await db
+ .update(heartbeatRuns)
+ .set({
+ contextSnapshot: context,
+ updatedAt: new Date(),
+ })
+ .where(eq(heartbeatRuns.id, run.id));
+ }
+ const environmentAcquireStartedAtMs = Date.now();
+ let acquiredEnvironment: Awaited<
+ ReturnType<typeof envOrchestrator.acquireForRun>
+ >;
+ try {
+ await controllerLease.assertOwned();
+ const remoteRecovery = runOptions.nativeRestartRecovery?.kind === "reattach_remote_runner"
+ ? runOptions.nativeRestartRecovery : null;
+ const recoveryWorkspace = remoteRecovery
+ ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null;
+ if (remoteRecovery && (!recoveryWorkspace || remoteRecovery.runId !== run.id ||
+ recoveryWorkspace.providerLeaseId !== remoteRecovery.remote.providerLeaseId ||
+ recoveryWorkspace.remoteCwd !== remoteRecovery.remote.remoteCwd)) {
+ throw new Error("native_remote_recovery_lease_mismatch");
+ }
+ acquiredEnvironment = await envOrchestrator.acquireForRun({
+ companyId: agent.companyId,
+ selectedEnvironmentId,
+ localEnvironmentId: localEnvironment.id,
+ adapterType: agent.adapterType,
+ adapterConfig: parseObject(agent.adapterConfig),
+ admittedLifecycleMode: persistedNativeExecutionInput?.session.lifecyclePolicy.mode,
+ issueId: issueId ?? null,
+ heartbeatRunId: run.id,
+ agentId: agent.id,
+ persistedExecutionWorkspace,
+ executionWorkspaceSettings: environmentExecutionWorkspaceSettings,
+ ...(remoteRecovery && recoveryWorkspace ? { reattachRemoteLease: {
+ leaseId: recoveryWorkspace.leaseId,
+ providerLeaseId: remoteRecovery.remote.providerLeaseId,
+ remoteCwd: remoteRecovery.remote.remoteCwd,
+ } } : {}),
+ });
+ await controllerLease.assertOwned();
+ nativeRunnerPreparationSpans.push({
+ name: "environment.acquire",
+ parentName: "task.run",
+ startedAtMs: environmentAcquireStartedAtMs,
+ endedAtMs: Date.now(),
+ attributes: { adapter: agent.adapterType },
+ });
+ } catch (error) {
+ nativeRunnerPreparationSpans.push({
+ name: "environment.acquire",
+ parentName: "task.run",
+ startedAtMs: environmentAcquireStartedAtMs,
+ endedAtMs: Date.now(),
+ outcome: "failed",
+ attributes: { adapter: agent.adapterType },
+ });
+ throw error;
+ }
+ const selectedEnvironment = acquiredEnvironment.environment;
+ // Defense-in-depth: re-check the actually-acquired environment against the
+ // execution allowlist. Even if selection were bypassed, a denied (local/ssh/
+ // non-k8s) environment FAILS the run here rather than executing untrusted.
+ const allowlistDecision = evaluateExecutionAllowlist(executionPolicy, {
+ driver: selectedEnvironment.driver,
+ provider:
+ typeof selectedEnvironment.config?.provider === "string"
+ ? selectedEnvironment.config.provider
+ : null,
+ });
+ if (!allowlistDecision.allowed) {
+ logger.error(
+ {
+ runId: run.id,
+ issueId,
+ agentId: agent.id,
+ environmentId: selectedEnvironment.id,
+ deniedDriver: allowlistDecision.deniedDriver,
+ deniedProvider: allowlistDecision.deniedProvider,
+ },
+ "Execution allowlist denied the resolved environment; failing run",
+ );
+ throw new Error(allowlistDecision.reason);
+ }
+ let activeEnvironmentLease = {
+ environment: acquiredEnvironment.environment,
+ lease: acquiredEnvironment.lease,
+ leaseContext: acquiredEnvironment.leaseContext,
+ };
+ const duplexObservabilityRecorder = createHostDuplexObservabilityRecorder(
+ {
+ tracer: getStartupTracer(),
+ incrementCounter: (metric) => {
+ void incrementToolRuntimeMetricCounter(db, {
+ companyId: run.companyId,
+ metric,
+ }).catch(() => {});
+ },
+ emitTransportEvent: (event) => {
+ void (async () => {
+ await appendRunEvent(run, {
+ eventType: event.name,
+ stream: "system",
+ level: event.dimensions.outcome === "error" ? "warn" : "info",
+ payload: { ...event.dimensions },
+ });
+ })().catch(() => {});
+ },
+ },
+ );
+ const environmentRealizeStartedAtMs = Date.now();
+ let realizationResult: Awaited<
+ ReturnType<typeof envOrchestrator.realizeForRun>
+ >;
+ try {
+ realizationResult = await envOrchestrator.realizeForRun({
+ environment: selectedEnvironment,
+ lease: activeEnvironmentLease.lease,
+ adapterType: agent.adapterType,
+ companyId: agent.companyId,
+ issueId: issueId ?? null,
+ heartbeatRunId: run.id,
+ executionWorkspace,
+ effectiveExecutionWorkspaceMode,
+ persistedExecutionWorkspace,
+ duplexObservabilityRecorder,
+ });
+ nativeRunnerPreparationSpans.push({
+ name: "environment.workspace.realize",
+ parentName: "task.run",
+ startedAtMs: environmentRealizeStartedAtMs,
+ endedAtMs: Date.now(),
+ attributes: { driver: selectedEnvironment.driver },
+ });
+ } catch (error) {
+ nativeRunnerPreparationSpans.push({
+ name: "environment.workspace.realize",
+ parentName: "task.run",
+ startedAtMs: environmentRealizeStartedAtMs,
+ endedAtMs: Date.now(),
+ outcome: "failed",
+ attributes: { driver: selectedEnvironment.driver },
+ });
+ throw error;
+ }
+ const environmentRealizeEndedAtMs = Date.now();
+ activeEnvironmentLease = {
+ ...activeEnvironmentLease,
+ lease: realizationResult.lease,
+ };
+ persistedExecutionWorkspace =
+ realizationResult.persistedExecutionWorkspace;
+ // A sandbox realization may materialize or replace the durable workspace
+ // after the host-side provisioning boundary above. Bind that final ID to
+ // the issue before dispatch so warm turns reuse the exact same workspace
+ // and lease scope instead of silently creating a per-run replacement.
+ await bindIssueToPersistedExecutionWorkspace(persistedExecutionWorkspace);
+ const workspaceRealization = realizationResult.workspaceRealization;
+ const executionTarget = realizationResult.executionTarget;
+ // Preserve the host-owned source before adapter context can share lease
+ // metadata. A later copy-back failure must not adopt a rebound source.
+ const workspaceRestoreSource = structuredClone(realizationResult.lease);
+ return {
+ resolvedWorkspace,
+ hostExecutionWorkspaceConfig,
+ latestWorkspaceConfigMetadata,
+ workspaceConfigFreshness,
+ workspaceOperationRecorder,
+ executionWorkspace,
+ reusedExecutionWorkspace,
+ resolvedWorkspaceReusePolicy,
+ persistedExecutionWorkspace,
+ projectRepositoryPaths,
+ environmentAcquireStartedAtMs,
+ selectedEnvironment,
+ activeEnvironmentLease,
+ realizationResult,
+ environmentRealizeEndedAtMs,
+ workspaceRealization,
+ executionTarget,
+ workspaceRestoreSource,
+ };
+}
diff --git a/tests/e2e/composer-stop.spec.ts b/tests/e2e/composer-stop.spec.ts
index e5030d6f7b..46cc882fe1 100644
--- a/tests/e2e/composer-stop.spec.ts
+++ b/tests/e2e/composer-stop.spec.ts
@@ -205,6 +205,9 @@ for (const adapter of ["process", "paperclip_runner"] as const) {
"-e",
"console.log('stop fixture ready'); setInterval(() => console.log('working'), 200);",
],
+ // The child only runs this script; server-only fixture
+ // loaders must not follow NODE_OPTIONS into the adapter.
+ env: { NODE_OPTIONS: "" },
graceSec: 1,
}
: { provider: "codex", model: "gpt-5.1-codex-mini" },
diff --git a/tests/e2e/workspace-bootstrap/README.md b/tests/e2e/workspace-bootstrap/README.md
index 29f9458a8b..3bf25b641e 100644
--- a/tests/e2e/workspace-bootstrap/README.md
+++ b/tests/e2e/workspace-bootstrap/README.md
@@ -9,14 +9,13 @@ timeout only when the test explicitly arms a marker inside its repository.
From a development worktree, start the foreground instance:
```sh
-BOOTSTRAP_FIXTURE_KEY=not-a-provider-key \
-PATH="$PWD/tests/e2e/workspace-bootstrap/bin:$PATH" \
-NODE_ENV=test \
-node cli/node_modules/tsx/dist/cli.mjs cli/src/index.ts test-drive \
- --harness codex --api-key-env BOOTSTRAP_FIXTURE_KEY \
- --company-name 'Workspace Recovery QA' --no-browser
+node cli/node_modules/tsx/dist/cli.mjs tests/e2e/workspace-bootstrap/launch.ts
```
+The launcher creates an isolated test-drive, installs the Git shim, and sets a
+10-second snapshot deadline after test-drive clears inherited configuration.
+Production's longer snapshot deadline would outlast the injected failure.
+
Set `BOOTSTRAP_REAL_GIT` to the absolute Git executable if it is not
`/usr/bin/git`. Do not point this fixture at a normal development or production
instance. The seeder checks the loopback host, trusted-local mode, and company
diff --git a/tests/e2e/workspace-bootstrap/launch.ts b/tests/e2e/workspace-bootstrap/launch.ts
new file mode 100644
index 0000000000..c05925c76a
--- /dev/null
+++ b/tests/e2e/workspace-bootstrap/launch.ts
@@ -0,0 +1,18 @@
+import path from "node:path";
+import { prepareTestDriveEnvironment, testDriveCommand } from "../../../cli/src/commands/test-drive.js";
+
+const options = {
+ harness: "codex",
+ apiKeyEnv: "BOOTSTRAP_FIXTURE_KEY",
+ companyName: "Workspace Recovery QA",
+ browser: false,
+} as const;
+
+process.env.BOOTSTRAP_FIXTURE_KEY = "not-a-provider-key";
+await prepareTestDriveEnvironment(options);
+process.env.NODE_ENV = "test";
+// Test-drive clears PAPERCLIP_* settings. Apply the fixture's short deadline
+// after isolation; production snapshots may legitimately take much longer.
+process.env.PAPERCLIP_WORKSPACE_GIT_SNAPSHOT_TIMEOUT_MS = "10000";
+process.env.PATH = `${path.join(import.meta.dirname, "bin")}${path.delimiter}${process.env.PATH ?? ""}`;
+await testDriveCommand(options);
diff --git a/tests/e2e/workspace-bootstrap/recovery.spec.ts b/tests/e2e/workspace-bootstrap/recovery.spec.ts
index f8ac233e81..884a20a8bd 100644
--- a/tests/e2e/workspace-bootstrap/recovery.spec.ts
+++ b/tests/e2e/workspace-bootstrap/recovery.spec.ts
@@ -22,11 +22,11 @@ for (const persistent of [false, true]) {
if (await announcement.isVisible()) await announcement.click();
await page.getByRole("link", { name: "Tasks", exact: true }).click();
await page.getByRole("button", { name: "New Task", exact: true }).last().click();
- await page.getByPlaceholder("Task title (optional)", { exact: true }).fill(title);
- await page.getByRole("button", { name: "Assignee", exact: true }).click();
- await page.getByRole("button", { name: fixture.agentName, exact: true }).click();
+ await page.getByRole("textbox", { name: "editable markdown" }).fill(title);
+ await page.getByRole("button", { name: "Select assignee", exact: true }).click();
+ await page.getByRole("option").filter({ hasText: fixture.agentName }).click();
// Let the closing popover unmount before clicking another popover trigger.
- await expect(page.getByRole("textbox", { name: "Search assignees...", includeHidden: true })).toHaveCount(0);
+ await expect(page.getByRole("searchbox", { name: "Search assignees", includeHidden: true })).toHaveCount(0);
// The preceding popover's focus restoration can consume the first click.
await expect(async () => {
if (!await page.getByRole("textbox", { name: "Search projects..." }).isVisible()) {
@@ -36,15 +36,21 @@ for (const persistent of [false, true]) {
}).toPass({ timeout: 10_000, intervals: [1_000] });
await page.getByRole("button", { name: fixture.projectName, exact: true }).click();
await expect(page.getByRole("textbox", { name: "Search projects...", includeHidden: true })).toHaveCount(0);
- await page.getByRole("button", { name: "Create Task", exact: true }).click();
- const taskLink = page.getByRole("complementary").getByRole("link", { name: title, exact: true });
+ const created = page.waitForResponse(response =>
+ response.request().method() === "POST"
+ && new URL(response.url()).pathname === `/api/companies/${fixture.companyId}/issues`,
+ );
+ await page.getByRole("button", { name: "Create task", exact: true }).click();
+ const response = await created;
+ expect(response.ok(), await response.text()).toBeTruthy();
+ const task = await response.json();
+ expect(task.description).toBe(title);
+ const taskLink = page.getByRole("complementary").getByRole("link", { name: task.title, exact: true });
await expect(taskLink).toBeVisible();
// Follow the UI's actual persisted link, including across task-tab state.
const href = await taskLink.getAttribute("href");
expect(href).toBeTruthy();
await page.goto(new URL(href!, base).href);
- const tasks = await api(`/companies/${fixture.companyId}/issues`);
- const task = tasks.find((row: { title: string }) => row.title === title);
const runs = async () => (await api(`/companies/${fixture.companyId}/heartbeat-runs`)).filter((row: { agentId: string }) => row.agentId === fixture.agentId);
await expect.poll(async () => (await runs()).some((row: { errorCode: string }) => row.errorCode === "workspace_git_scan_timeout"), { timeout: 30_000 }).toBe(true);
await expect(page.getByText(/Agent resumes in/)).toBeVisible();