The open-source agent harness - the runtime layer that turns an LLM into a working agent.
feat: reactivate inactive snapshots on Daytona sandbox create and raise P1 alerts for unexpected failures (#1005)
Sujai Kumar Guptaf9755751a7c4
diff --git a/.changeset/daytona-inactive-snapshot-reactivate.md b/.changeset/daytona-inactive-snapshot-reactivate.mdnew file mode 100644index 000000000..1bb7f296b--- /dev/null+++ b/.changeset/daytona-inactive-snapshot-reactivate.md@@ -0,0 +1,6 @@+---+'@truefoundry/trueforge-core': patch+'@truefoundry/trueforge': patch+---++On Daytona sandbox create, reactivate inactive snapshots with a clear retry error, and raise a P1 for unexpected create failures (not authz).diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.tsindex ab1bb2b06..c8376d311 100644--- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts+++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts@@ -35,9 +35,13 @@ const SANDBOX_STATE_STARTED = 'started';const SNAPSHOT_CONFLICT_STATUS = 409;const BUILD_STATE_ACTIVE = 'active';+const BUILD_STATE_INACTIVE = 'inactive';const BUILD_STATE_ERROR = 'error';const BUILD_STATE_BUILD_FAILED = 'build_failed';+const SNAPSHOT_INACTIVE_USER_MESSAGE =+ 'The sandbox is temporarily unavailable and is being restored. Please try again in a few minutes.';+/** Same default the Daytona SDK applies when `DaytonaConfig.apiUrl` is omitted. */const DEFAULT_DAYTONA_API_URL = 'https://app.daytona.io/api';@@ -77,6 +81,8 @@ export interface DaytonaSandboxProviderOptions {/** Defaults to 1 hour (same as the gateway's max agent execution time). */previewUrlExpirySeconds?: number;logger: Logger;+ /** Host alert hook (e.g. Sentry P1). */+ onCriticalAlert?: (error: Error) => void;}export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnvironment> {@@ -93,6 +99,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnvprivate readonly apiKey: string;private readonly apiUrl: string;private readonly logger: Logger;+ private readonly onCriticalAlert: ((error: Error) => void) | undefined;private readonly daytona: Daytona;private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();// De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip.@@ -110,6 +117,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnvthis.fileMaxBytesForDownload = options.fileMaxBytesForDownload;this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT;this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS;+ this.onCriticalAlert = options.onCriticalAlert;this.logger = options.logger.child({ module: 'DaytonaProvider' });}@@ -322,6 +330,36 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv});}+ /** Activate an inactive tip, then throw so create is not attempted. Other states no-op. */+ private async rejectIfSnapshotInactive(snapshotRef: string): Promise<void> {+ const snapshot = await this.getSnapshot(snapshotRef);+ if (snapshot?.state !== BUILD_STATE_INACTIVE) {+ return;+ }+ await this.daytona.snapshot.activate(snapshot);+ throw new Error(SNAPSHOT_INACTIVE_USER_MESSAGE);+ }++ /** Fresh create: reject an inactive tip before calling Daytona create. */+ private async createFreshSandbox(environment?: DaytonaSandboxEnvironment): Promise<Sandbox> {+ const env = this.requireEnvironment(environment);+ try {+ await this.rejectIfSnapshotInactive(env.snapshot_ref);+ return await this.daytona.create(this.buildCreateParams(env));+ } catch (error) {+ if (error instanceof DaytonaError && (error.statusCode === 401 || error.statusCode === 403)) {+ throw error;+ }+ const alertError = error instanceof Error ? error : new Error('Daytona sandbox create failed', { cause: error });+ this.logger.error('Daytona sandbox create failed', {+ snapshot_ref: env.snapshot_ref,+ ...extractErrorLogFields(alertError),+ });+ this.onCriticalAlert?.(alertError);+ throw error;+ }+ }+private async getOrCreateSandbox(sandboxId?: string,environment?: DaytonaSandboxEnvironment,@@ -336,7 +374,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnvconst sandbox = sandboxId? await this.restoreExistingSandbox(sandboxId)- : await this.daytona.create(this.buildCreateParams(environment));+ : await this.createFreshSandbox(environment);const entry = { sandbox, defaultTimeoutMs: this.timeoutMs };DaytonaSandboxProvider.cachedSandboxes.set(sandbox.name, entry);diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.tsindex 88b3f8d13..cbf74bac4 100644--- a/packages/trueforge/src/sandbox/providerUtils.ts+++ b/packages/trueforge/src/sandbox/providerUtils.ts@@ -16,6 +16,7 @@ import {type SandboxStatus,type StoredSandboxProviderManifest,} from '../schemas/sandboxProvider';+import { captureCriticalException } from '../sentry';/** Bound Daytona RPCs so API requests and controller ticks cannot hang indefinitely. */export const DAYTONA_RPC_TIMEOUT_MS = 5_000;@@ -103,6 +104,11 @@ export function toDaytonaSandboxProvider({tenantName: tenant_id,fileMaxBytesForDownload: configuration.SANDBOX_FILE_MAX_BYTES_FOR_DOWNLOAD,logger,+ onCriticalAlert: error => {+ captureCriticalException(error, {+ tags: { module: 'DaytonaProvider', operation: 'sandboxCreate', priority: 'P1' },+ });+ },});}