Skip to content

The open-source app everyone uses to manage agents at work

Fix runtime state race, workspace sync, plugin startup, and orphaned leases (#4804)

Devin Foley4cf612a92dc7
diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts
index 7b96e2551e..aa5e1afd1f 100644
--- a/server/src/__tests__/environment-runtime.test.ts
+++ b/server/src/__tests__/environment-runtime.test.ts
@@ -700,6 +700,98 @@ describeEmbeddedPostgres("environmentRuntimeService", () => {
}));
});
+ it("waits briefly for a ready sandbox provider plugin worker to come online", async () => {
+ const pluginId = randomUUID();
+ const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
+ const providerConfig = {
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: false,
+ };
+ const environment = {
+ ...baseEnvironment,
+ name: "Eventually Running Plugin Sandbox",
+ driver: "sandbox",
+ config: providerConfig,
+ };
+ await environmentService(db).update(environment.id, {
+ driver: "sandbox",
+ name: environment.name,
+ config: providerConfig,
+ });
+ await db.insert(plugins).values({
+ id: pluginId,
+ pluginKey: "acme.eventually-running-sandbox-provider",
+ packageName: "@acme/eventually-running-sandbox-provider",
+ version: "1.0.0",
+ apiVersion: 1,
+ categories: ["automation"],
+ manifestJson: {
+ id: "acme.eventually-running-sandbox-provider",
+ apiVersion: 1,
+ version: "1.0.0",
+ displayName: "Eventually Running Sandbox Provider",
+ description: "Test plugin worker startup grace period",
+ author: "Acme",
+ categories: ["automation"],
+ capabilities: ["environment.drivers.register"],
+ entrypoints: { worker: "dist/worker.js" },
+ environmentDrivers: [
+ {
+ driverKey: "fake-plugin",
+ kind: "sandbox_provider",
+ displayName: "Fake Plugin",
+ configSchema: { type: "object" },
+ },
+ ],
+ },
+ status: "ready",
+ installOrder: 1,
+ updatedAt: new Date(),
+ } as any);
+
+ let runningChecks = 0;
+ const workerManager = {
+ isRunning: vi.fn((id: string) => {
+ if (id !== pluginId) return false;
+ runningChecks += 1;
+ return runningChecks >= 3;
+ }),
+ call: vi.fn(async (_pluginId: string, method: string) => {
+ if (method === "environmentAcquireLease") {
+ return {
+ providerLeaseId: "sandbox-1",
+ metadata: {
+ provider: "fake-plugin",
+ image: "fake:test",
+ timeoutMs: 1234,
+ reuseLease: false,
+ },
+ };
+ }
+ throw new Error(`Unexpected plugin method: ${method}`);
+ }),
+ } as unknown as PluginWorkerManager;
+ const runtimeWithPlugin = environmentRuntimeService(db, {
+ pluginWorkerManager: workerManager,
+ pluginWorkerReadyTimeoutMs: 25,
+ pluginWorkerReadyPollMs: 1,
+ });
+
+ const acquired = await runtimeWithPlugin.acquireRunLease({
+ companyId,
+ environment,
+ issueId: null,
+ heartbeatRunId: runId,
+ persistedExecutionWorkspace: null,
+ });
+
+ expect(acquired.lease.providerLeaseId).toBe("sandbox-1");
+ expect(workerManager.isRunning).toHaveBeenCalledTimes(3);
+ expect(workerManager.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.anything());
+ });
+
it("falls back to acquire when plugin-backed sandbox lease resume throws", async () => {
const pluginId = randomUUID();
const { companyId, environment: baseEnvironment, runId } = await seedEnvironment();
diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts
index 838b34b661..d66c2617ea 100644
--- a/server/src/__tests__/heartbeat-process-recovery.test.ts
+++ b/server/src/__tests__/heartbeat-process-recovery.test.ts
@@ -14,6 +14,8 @@ import {
createDb,
documentRevisions,
documents,
+ environmentLeases,
+ environments,
heartbeatRunEvents,
heartbeatRuns,
issueComments,
@@ -309,6 +311,8 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
await db.delete(agentRuntimeState);
await db.delete(companySkills);
await db.delete(costEvents);
+ await db.delete(environmentLeases);
+ await db.delete(environments);
await db.delete(issueComments);
await db.delete(issueDocuments);
await db.delete(documentRevisions);
@@ -466,6 +470,48 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
return { companyId, agentId, runId, wakeupRequestId, issueId };
}
+ async function seedEnvironmentLeaseFixture(input: {
+ companyId: string;
+ runId: string;
+ issueId: string;
+ provider?: string;
+ }) {
+ const environmentId = randomUUID();
+ const leaseId = randomUUID();
+ const now = new Date("2026-03-19T00:00:00.000Z");
+
+ await db.insert(environments).values({
+ id: environmentId,
+ companyId: input.companyId,
+ name: "Local test environment",
+ driver: "local",
+ status: "active",
+ config: {},
+ metadata: null,
+ });
+
+ await db.insert(environmentLeases).values({
+ id: leaseId,
+ companyId: input.companyId,
+ environmentId,
+ issueId: input.issueId,
+ heartbeatRunId: input.runId,
+ status: "active",
+ leasePolicy: "ephemeral",
+ provider: input.provider ?? "local",
+ providerLeaseId: null,
+ acquiredAt: now,
+ lastUsedAt: now,
+ metadata: {
+ driver: "local",
+ },
+ createdAt: now,
+ updatedAt: now,
+ });
+
+ return { environmentId, leaseId };
+ }
+
async function seedStrandedIssueFixture(input: {
status: "todo" | "in_progress";
runStatus: "failed" | "timed_out" | "cancelled" | "succeeded";
@@ -877,6 +923,30 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
expect(issue?.checkoutRunId).toBe(runId);
});
+ it("releases active environment leases when an orphaned run is reaped", async () => {
+ const { runId, issueId, companyId } = await seedRunFixture({
+ processPid: 999_999_999,
+ });
+ const { leaseId } = await seedEnvironmentLeaseFixture({
+ companyId,
+ runId,
+ issueId,
+ });
+ const heartbeat = heartbeatService(db);
+
+ const result = await heartbeat.reapOrphanedRuns();
+ expect(result.reaped).toBe(1);
+ expect(result.runIds).toEqual([runId]);
+
+ const lease = await db
+ .select()
+ .from(environmentLeases)
+ .where(eq(environmentLeases.id, leaseId))
+ .then((rows) => rows[0] ?? null);
+ expect(lease?.status).toBe("failed");
+ expect(lease?.releasedAt).toBeTruthy();
+ });
+
it.skipIf(process.platform === "win32")("reaps orphaned descendant process groups when the parent pid is already gone", async () => {
const orphan = await spawnOrphanedProcessGroup();
cleanupPids.add(orphan.descendantPid);
diff --git a/server/src/__tests__/heartbeat-runtime-state.test.ts b/server/src/__tests__/heartbeat-runtime-state.test.ts
new file mode 100644
index 0000000000..5a8faa9a55
--- /dev/null
+++ b/server/src/__tests__/heartbeat-runtime-state.test.ts
@@ -0,0 +1,88 @@
+import { randomUUID } from "node:crypto";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+import { eq } from "drizzle-orm";
+import {
+ agents,
+ agentRuntimeState,
+ agentWakeupRequests,
+ companies,
+ createDb,
+ heartbeatRunEvents,
+ heartbeatRuns,
+} from "@paperclipai/db";
+import {
+ getEmbeddedPostgresTestSupport,
+ startEmbeddedPostgresTestDatabase,
+} from "./helpers/embedded-postgres.js";
+import { heartbeatService } from "../services/heartbeat.ts";
+
+const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport();
+const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip;
+
+if (!embeddedPostgresSupport.supported) {
+ console.warn(
+ `Skipping embedded Postgres heartbeat runtime-state tests on this host: ${embeddedPostgresSupport.reason ?? "unsupported environment"}`,
+ );
+}
+
+describeEmbeddedPostgres("heartbeat runtime state deduplication", () => {
+ let db!: ReturnType<typeof createDb>;
+ let tempDb: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>> | null = null;
+
+ beforeAll(async () => {
+ tempDb = await startEmbeddedPostgresTestDatabase("heartbeat-runtime-state-");
+ db = createDb(tempDb.connectionString);
+ }, 20_000);
+
+ afterEach(async () => {
+ await db.delete(heartbeatRunEvents);
+ await db.delete(heartbeatRuns);
+ await db.delete(agentWakeupRequests);
+ await db.delete(agentRuntimeState);
+ await db.delete(agents);
+ await db.delete(companies);
+ });
+
+ afterAll(async () => {
+ await tempDb?.cleanup();
+ });
+
+ it("deduplicates concurrent runtime-state creation", async () => {
+ const companyId = randomUUID();
+ const agentId = randomUUID();
+ const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
+
+ await db.insert(companies).values({
+ id: companyId,
+ name: "Paperclip",
+ issuePrefix,
+ requireBoardApprovalForNewAgents: false,
+ });
+
+ await db.insert(agents).values({
+ id: agentId,
+ companyId,
+ name: "CodexCoder",
+ role: "engineer",
+ status: "idle",
+ adapterType: "codex_local",
+ adapterConfig: {},
+ runtimeConfig: {},
+ permissions: {},
+ });
+
+ const heartbeat = heartbeatService(db);
+ const results = await Promise.all(Array.from({ length: 12 }, () => heartbeat.getRuntimeState(agentId)));
+
+ expect(results.every((row) => row?.agentId === agentId)).toBe(true);
+
+ const rows = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, agentId));
+ expect(rows).toHaveLength(1);
+ expect(rows[0]).toMatchObject({
+ agentId,
+ companyId,
+ adapterType: "codex_local",
+ stateJson: {},
+ });
+ });
+});
diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts
index 0922327654..ad11eecdd7 100644
--- a/server/src/__tests__/issues-service.test.ts
+++ b/server/src/__tests__/issues-service.test.ts
@@ -2116,6 +2116,109 @@ describeEmbeddedPostgres("issueService.create workspace inheritance", () => {
mode: "operator_branch",
});
});
+
+ it("syncs reused execution workspace config when issue workspace settings are updated", async () => {
+ const companyId = randomUUID();
+ const projectId = randomUUID();
+ const projectWorkspaceId = randomUUID();
+ const executionWorkspaceId = randomUUID();
+ const issueId = randomUUID();
+
+ await db.insert(companies).values({
+ id: companyId,
+ name: "Paperclip",
+ issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`,
+ requireBoardApprovalForNewAgents: false,
+ });
+ await instanceSettingsService(db).updateExperimental({ enableIsolatedWorkspaces: true });
+
+ await db.insert(projects).values({
+ id: projectId,
+ companyId,
+ name: "Workspace project",
+ status: "in_progress",
+ });
+
+ await db.insert(projectWorkspaces).values({
+ id: projectWorkspaceId,
+ companyId,
+ projectId,
+ name: "Primary workspace",
+ });
+
+ await db.insert(executionWorkspaces).values({
+ id: executionWorkspaceId,
+ companyId,
+ projectId,
+ projectWorkspaceId,
+ mode: "isolated_workspace",
+ strategyType: "git_worktree",
+ name: "Issue worktree",
+ status: "active",
+ providerType: "git_worktree",
+ metadata: {
+ config: {
+ environmentId: "env-old",
+ provisionCommand: "bash ./scripts/provision-old.sh",
+ teardownCommand: "bash ./scripts/teardown-old.sh",
+ workspaceRuntime: { profile: "old" },
+ },
+ },
+ });
+
+ await db.insert(issues).values({
+ id: issueId,
+ companyId,
+ projectId,
+ projectWorkspaceId,
+ title: "Recovery issue",
+ status: "in_progress",
+ priority: "medium",
+ executionWorkspaceId,
+ executionWorkspacePreference: "reuse_existing",
+ executionWorkspaceSettings: {
+ mode: "isolated_workspace",
+ environmentId: "env-old",
+ workspaceStrategy: {
+ type: "git_worktree",
+ provisionCommand: "bash ./scripts/provision-old.sh",
+ teardownCommand: "bash ./scripts/teardown-old.sh",
+ },
+ workspaceRuntime: { profile: "old" },
+ },
+ });
+
+ await svc.update(issueId, {
+ executionWorkspaceSettings: {
+ mode: "isolated_workspace",
+ environmentId: "env-new",
+ workspaceStrategy: {
+ type: "cloud_sandbox",
+ provisionCommand: "bash ./scripts/provision-new.sh",
+ teardownCommand: "bash ./scripts/teardown-new.sh",
+ },
+ workspaceRuntime: { profile: "new" },
+ },
+ });
+
+ const workspace = await db
+ .select({ metadata: executionWorkspaces.metadata })
+ .from(executionWorkspaces)
+ .where(eq(executionWorkspaces.id, executionWorkspaceId))
+ .then((rows) => rows[0] ?? null);
+
+ expect(workspace?.metadata).toEqual({
+ config: {
+ environmentId: "env-new",
+ provisionCommand: "bash ./scripts/provision-new.sh",
+ teardownCommand: "bash ./scripts/teardown-new.sh",
+ cleanupCommand: null,
+ workspaceRuntime: { profile: "new" },
+ desiredState: null,
+ serviceStates: null,
+ },
+ });
+ });
});
describeEmbeddedPostgres("issueService.findMentionedProjectIds", () => {
diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts
index b403d5f907..92023f432e 100644
--- a/server/src/services/environment-runtime.ts
+++ b/server/src/services/environment-runtime.ts
@@ -151,6 +151,13 @@ export interface EnvironmentRuntimeLeaseRecord {
leaseContext: ReturnType<typeof buildEnvironmentLeaseContext>;
}
+const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS = 5_000;
+const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS = 100;
+
+function delay(ms: number): Promise<void> {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
function getLeaseDriverKey(lease: Pick<EnvironmentLease, "metadata">, environment: Pick<Environment, "driver">): string {
const leaseDriver = typeof lease.metadata?.driver === "string" ? lease.metadata.driver : null;
return leaseDriver ?? environment.driver;
@@ -288,10 +295,63 @@ function createSshEnvironmentDriver(db: Db): EnvironmentRuntimeDriver {
function createSandboxEnvironmentDriver(
db: Db,
- pluginWorkerManager?: PluginWorkerManager,
+ options: {
+ pluginWorkerManager?: PluginWorkerManager;
+ pluginWorkerReadyTimeoutMs?: number;
+ pluginWorkerReadyPollMs?: number;
+ } = {},
): EnvironmentRuntimeDriver {
+ const pluginWorkerManager = options.pluginWorkerManager;
+ const pluginWorkerReadyTimeoutMs = options.pluginWorkerReadyTimeoutMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS;
+ const pluginWorkerReadyPollMs = options.pluginWorkerReadyPollMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS;
const environmentsSvc = environmentService(db);
+ async function resolveSandboxProviderPlugin(input: { provider: string }) {
+ const running = await resolvePluginSandboxProviderDriverByKey({
+ db,
+ driverKey: input.provider,
+ workerManager: pluginWorkerManager,
+ requireRunning: true,
+ });
+ if (running) {
+ return { state: "running" as const, resolved: running };
+ }
+
+ const installed = await resolvePluginSandboxProviderDriverByKey({
+ db,
+ driverKey: input.provider,
+ workerManager: pluginWorkerManager,
+ requireRunning: false,
+ });
+ if (!installed) {
+ return { state: "missing" as const, resolved: null };
+ }
+
+ if (installed.plugin.status !== "ready") {
+ return { state: "not_ready" as const, resolved: installed };
+ }
+
+ if (!pluginWorkerManager) {
+ return { state: "worker_unavailable" as const, resolved: installed };
+ }
+
+ const deadline = Date.now() + Math.max(0, pluginWorkerReadyTimeoutMs);
+ while (Date.now() < deadline) {
+ const retried = await resolvePluginSandboxProviderDriverByKey({
+ db,
+ driverKey: input.provider,
+ workerManager: pluginWorkerManager,
+ requireRunning: true,
+ });
+ if (retried) {
+ return { state: "running" as const, resolved: retried };
+ }
+ await delay(Math.max(1, pluginWorkerReadyPollMs));
+ }
+
+ return { state: "worker_unavailable" as const, resolved: installed };
+ }
+
async function resolvePluginSandboxRuntimeConfig(input: {
environment: Environment;
lease: EnvironmentLease;
@@ -342,17 +402,29 @@ function createSandboxEnvironmentDriver(
// Check if this provider should be handled by a plugin.
if (!isBuiltinSandboxProvider(parsed.config.provider)) {
- const pluginProvider = await resolvePluginSandboxProviderDriverByKey({
- db,
- driverKey: parsed.config.provider,
- workerManager: pluginWorkerManager,
- requireRunning: true,
+ const pluginProvider = await resolveSandboxProviderPlugin({
+ provider: parsed.config.provider,
});
- if (!pluginProvider || !pluginWorkerManager) {
+ if (pluginProvider.state === "missing") {
throw new Error(
`Sandbox provider "${parsed.config.provider}" is not registered as a built-in provider and no matching plugin is available.`,
);
}
+ if (pluginProvider.state === "not_ready") {
+ throw new Error(
+ `Sandbox provider "${parsed.config.provider}" is installed via plugin "${pluginProvider.resolved.plugin.pluginKey}", but that plugin is currently ${pluginProvider.resolved.plugin.status}.`,
+ );
+ }
+ if (pluginProvider.state === "worker_unavailable") {
+ throw new Error(
+ `Sandbox provider "${parsed.config.provider}" is installed via plugin "${pluginProvider.resolved.plugin.pluginKey}", but its worker is not running.`,
+ );
+ }
+ if (!pluginWorkerManager) {
+ throw new Error(
+ `Sandbox provider "${parsed.config.provider}" is installed, but sandbox plugin workers are unavailable in this server process.`,
+ );
+ }
const workerConfig = stripSandboxProviderEnvelope(parsed.config);
const storedConfig = storedParsed.config;
@@ -368,7 +440,7 @@ function createSandboxEnvironmentDriver(
const providerLease = reusableLease?.providerLeaseId
? await pluginWorkerManager.call(
- pluginProvider.plugin.id,
+ pluginProvider.resolved.plugin.id,
"environmentResumeLease",
{
driverKey: parsed.config.provider,
@@ -385,7 +457,7 @@ function createSandboxEnvironmentDriver(
).catch(() => null)
: null;
const acquiredLease = providerLease ?? await pluginWorkerManager.call(
- pluginProvider.plugin.id,
+ pluginProvider.resolved.plugin.id,
"environmentAcquireLease",
{
driverKey: parsed.config.provider,
@@ -414,13 +486,13 @@ function createSandboxEnvironmentDriver(
metadata: {
driver: input.environment.driver,
executionWorkspaceMode: input.executionWorkspaceMode,
- pluginId: pluginProvider.plugin.id,
- pluginKey: pluginProvider.plugin.pluginKey,
+ pluginId: pluginProvider.resolved.plugin.id,
+ pluginKey: pluginProvider.resolved.plugin.pluginKey,
sandboxProviderPlugin: true,
...sandboxConfigForLeaseMetadata(storedConfig),
...stripSecretRefValuesFromPluginLeaseMetadata({
metadata: acquiredLease.metadata,
- schema: pluginProvider.driver.configSchema as Record<string, unknown> | null | undefined,
+ schema: pluginProvider.resolved.driver.configSchema as Record<string, unknown> | null | undefined,
}),
},
});
@@ -929,6 +1001,8 @@ export function environmentRuntimeService(
options: {
drivers?: EnvironmentRuntimeDriver[];
pluginWorkerManager?: PluginWorkerManager;
+ pluginWorkerReadyTimeoutMs?: number;
+ pluginWorkerReadyPollMs?: number;
} = {},
) {
const environmentsSvc = environmentService(db);
@@ -937,7 +1011,11 @@ export function environmentRuntimeService(
const defaultDrivers = [
createLocalEnvironmentDriver(db),
createSshEnvironmentDriver(db),
- createSandboxEnvironmentDriver(db, options.pluginWorkerManager),
+ createSandboxEnvironmentDriver(db, {
+ pluginWorkerManager: options.pluginWorkerManager,
+ pluginWorkerReadyTimeoutMs: options.pluginWorkerReadyTimeoutMs,
+ pluginWorkerReadyPollMs: options.pluginWorkerReadyPollMs,
+ }),
...(options.pluginWorkerManager
? [createPluginEnvironmentDriver(db, options.pluginWorkerManager)]
: []),
diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts
index 15352a7928..25535ed63a 100644
--- a/server/src/services/heartbeat.ts
+++ b/server/src/services/heartbeat.ts
@@ -2008,6 +2008,31 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const productivityReviews = productivityReviewService(db, { enqueueWakeup });
let unsafeTextProjectionPromise: Promise<boolean> | null = null;
+ async function releaseEnvironmentLeasesForRun(input: {
+ runId: string;
+ companyId: string;
+ agentId: string;
+ status: string | null | undefined;
+ failureReason?: string | null;
+ }) {
+ const releaseResult = await envOrchestrator.releaseForRun({
+ heartbeatRunId: input.runId,
+ companyId: input.companyId,
+ agentId: input.agentId,
+ status: leaseReleaseStatusForRunStatus(input.status),
+ failureReason: input.failureReason ?? undefined,
+ }).catch((err) => {
+ logger.warn({ err, runId: input.runId }, "failed to release environment leases for heartbeat run");
+ return null;
+ });
+ for (const releaseError of releaseResult?.errors ?? []) {
+ logger.warn(
+ { err: releaseError.error, leaseId: releaseError.leaseId, runId: input.runId },
+ "failed to release environment lease for heartbeat run",
+ );
+ }
+ }
+
async function hasUnsafeTextProjectionDatabase() {
if (!unsafeTextProjectionPromise) {
unsafeTextProjectionPromise = db
@@ -2647,7 +2672,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
const existing = await getRuntimeState(agent.id);
if (existing) return existing;
- return db
+ const inserted = await db
.insert(agentRuntimeState)
.values({
agentId: agent.id,
@@ -2655,8 +2680,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
adapterType: agent.adapterType,
stateJson: {},
})
+ .onConflictDoNothing({
+ target: agentRuntimeState.agentId,
+ })
.returning()
- .then((rows) => rows[0]);
+ .then((rows) => rows[0] ?? null);
+ if (inserted) return inserted;
+
+ const ensured = await getRuntimeState(agent.id);
+ if (!ensured) {
+ throw new Error(`Failed to ensure runtime state for agent ${agent.id}`);
+ }
+ return ensured;
}
async function setRunStatus(
@@ -4448,6 +4483,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
if (!finalizedRun) finalizedRun = await getRun(run.id);
if (!finalizedRun) continue;
finalizedRun = await classifyAndPersistRunLiveness(finalizedRun, parseObject(finalizedRun.resultJson)) ?? finalizedRun;
+ await releaseEnvironmentLeasesForRun({
+ runId: finalizedRun.id,
+ companyId: finalizedRun.companyId,
+ agentId: finalizedRun.agentId,
+ status: finalizedRun.status,
+ failureReason: finalizedRun.error ?? undefined,
+ });
let retriedRun: typeof heartbeatRuns.$inferSelect | null = null;
if (shouldRetry) {
@@ -5894,22 +5936,13 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {})
await finalizeAgentStatus(run.agentId, "failed").catch(() => undefined);
} finally {
const latestRun = await getRun(run.id).catch(() => null);
- const releaseResult = await envOrchestrator.releaseForRun({
- heartbeatRunId: run.id,
+ await releaseEnvironmentLeasesForRun({
+ runId: run.id,
companyId: run.companyId,
agentId: run.agentId,
- status: leaseReleaseStatusForRunStatus(latestRun?.status),
+ status: latestRun?.status,
failureReason: latestRun?.error ?? undefined,
- }).catch((err) => {
- logger.warn({ err, runId: run.id }, "failed to release environment leases for heartbeat run");
- return null;
});
- for (const releaseError of releaseResult?.errors ?? []) {
- logger.warn(
- { err: releaseError.error, leaseId: releaseError.leaseId, runId: run.id },
- "failed to release environment lease for heartbeat run",
- );
- }
await releaseRuntimeServicesForRun(run.id).catch(() => undefined);
activeRunExecutions.delete(run.id);
await startNextQueuedRunForAgent(run.agentId);
diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts
index 104d12da2d..cce66d09c9 100644
--- a/server/src/services/issues.ts
+++ b/server/src/services/issues.ts
@@ -42,6 +42,7 @@ import {
parseIssueExecutionWorkspaceSettings,
parseProjectExecutionWorkspacePolicy,
} from "./execution-workspace-policy.js";
+import { mergeExecutionWorkspaceConfig } from "./execution-workspaces.js";
import { instanceSettingsService } from "./instance-settings.js";
import { redactCurrentUserText } from "../log-redaction.js";
import { resolveIssueGoalId, resolveNextIssueGoalId } from "./issue-goal-fallback.js";
@@ -91,6 +92,17 @@ function readStringFromRecord(record: unknown, key: string) {
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
}
+function buildReusedExecutionWorkspaceConfigPatchFromIssueSettings(
+ settings: ReturnType<typeof parseIssueExecutionWorkspaceSettings>,
+) {
+ return {
+ environmentId: settings?.environmentId ?? null,
+ provisionCommand: settings?.workspaceStrategy?.provisionCommand ?? null,
+ teardownCommand: settings?.workspaceStrategy?.teardownCommand ?? null,
+ workspaceRuntime: settings?.workspaceRuntime ?? null,
+ };
+}
+
export interface IssueFilters {
status?: string;
assigneeAgentId?: string;
@@ -2890,6 +2902,14 @@ export function issueService(db: Db) {
issueData.projectWorkspaceId !== undefined ? issueData.projectWorkspaceId : existing.projectWorkspaceId;
const nextExecutionWorkspaceId =
issueData.executionWorkspaceId !== undefined ? issueData.executionWorkspaceId : existing.executionWorkspaceId;
+ const nextExecutionWorkspacePreference =
+ issueData.executionWorkspacePreference !== undefined
+ ? issueData.executionWorkspacePreference
+ : existing.executionWorkspacePreference;
+ const nextExecutionWorkspaceSettings =
+ issueData.executionWorkspaceSettings !== undefined
+ ? parseIssueExecutionWorkspaceSettings(issueData.executionWorkspaceSettings)
+ : parseIssueExecutionWorkspaceSettings(existing.executionWorkspaceSettings);
if (nextProjectWorkspaceId) {
await assertValidProjectWorkspace(existing.companyId, nextProjectId, nextProjectWorkspaceId);
}
@@ -2963,6 +2983,37 @@ export function issueService(db: Db) {
tx,
);
}
+ if (
+ issueData.executionWorkspaceSettings !== undefined &&
+ nextExecutionWorkspaceId &&
+ nextExecutionWorkspacePreference === "reuse_existing"
+ ) {
+ const workspace = await tx
+ .select({
+ id: executionWorkspaces.id,
+ metadata: executionWorkspaces.metadata,
+ })
+ .from(executionWorkspaces)
+ .where(
+ and(
+ eq(executionWorkspaces.id, nextExecutionWorkspaceId),
+ eq(executionWorkspaces.companyId, existing.companyId),
+ ),
+ )
+ .then((rows: Array<{ id: string; metadata: unknown }>) => rows[0] ?? null);
+ if (workspace) {
+ await tx
+ .update(executionWorkspaces)
+ .set({
+ metadata: mergeExecutionWorkspaceConfig(
+ (workspace.metadata as Record<string, unknown> | null) ?? null,
+ buildReusedExecutionWorkspaceConfigPatchFromIssueSettings(nextExecutionWorkspaceSettings),
+ ),
+ updatedAt: new Date(),
+ })
+ .where(eq(executionWorkspaces.id, workspace.id));
+ }
+ }
const [enriched] = await withIssueLabels(tx, [updated]);
return enriched;
};