Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

feat: reactivate inactive snapshots on Daytona sandbox create and raise P1 alerts for unexpected failures (#1005)

Sujai Kumar Guptaf9755751a7c4
diff --git a/.changeset/daytona-inactive-snapshot-reactivate.md b/.changeset/daytona-inactive-snapshot-reactivate.md
new file mode 100644
index 000000000..1bb7f296b
--- /dev/null
+++ b/.changeset/daytona-inactive-snapshot-reactivate.md
@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge-core': patch
+'@truefoundry/trueforge': patch
+---
+
+On Daytona sandbox create, reactivate inactive snapshots with a clear retry error, and raise a P1 for unexpected create failures (not authz).
diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
index ab1bb2b06..c8376d311 100644
--- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
+++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts
@@ -35,9 +35,13 @@ const SANDBOX_STATE_STARTED = 'started';
const SNAPSHOT_CONFLICT_STATUS = 409;
const BUILD_STATE_ACTIVE = 'active';
+const BUILD_STATE_INACTIVE = 'inactive';
const BUILD_STATE_ERROR = 'error';
const BUILD_STATE_BUILD_FAILED = 'build_failed';
+const SNAPSHOT_INACTIVE_USER_MESSAGE =
+ 'The sandbox is temporarily unavailable and is being restored. Please try again in a few minutes.';
+
/** Same default the Daytona SDK applies when `DaytonaConfig.apiUrl` is omitted. */
const DEFAULT_DAYTONA_API_URL = 'https://app.daytona.io/api';
@@ -77,6 +81,8 @@ export interface DaytonaSandboxProviderOptions {
/** Defaults to 1 hour (same as the gateway's max agent execution time). */
previewUrlExpirySeconds?: number;
logger: Logger;
+ /** Host alert hook (e.g. Sentry P1). */
+ onCriticalAlert?: (error: Error) => void;
}
export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnvironment> {
@@ -93,6 +99,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
private readonly apiKey: string;
private readonly apiUrl: string;
private readonly logger: Logger;
+ private readonly onCriticalAlert: ((error: Error) => void) | undefined;
private readonly daytona: Daytona;
private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();
// De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip.
@@ -110,6 +117,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
this.fileMaxBytesForDownload = options.fileMaxBytesForDownload;
this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT;
this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS;
+ this.onCriticalAlert = options.onCriticalAlert;
this.logger = options.logger.child({ module: 'DaytonaProvider' });
}
@@ -322,6 +330,36 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
});
}
+ /** Activate an inactive tip, then throw so create is not attempted. Other states no-op. */
+ private async rejectIfSnapshotInactive(snapshotRef: string): Promise<void> {
+ const snapshot = await this.getSnapshot(snapshotRef);
+ if (snapshot?.state !== BUILD_STATE_INACTIVE) {
+ return;
+ }
+ await this.daytona.snapshot.activate(snapshot);
+ throw new Error(SNAPSHOT_INACTIVE_USER_MESSAGE);
+ }
+
+ /** Fresh create: reject an inactive tip before calling Daytona create. */
+ private async createFreshSandbox(environment?: DaytonaSandboxEnvironment): Promise<Sandbox> {
+ const env = this.requireEnvironment(environment);
+ try {
+ await this.rejectIfSnapshotInactive(env.snapshot_ref);
+ return await this.daytona.create(this.buildCreateParams(env));
+ } catch (error) {
+ if (error instanceof DaytonaError && (error.statusCode === 401 || error.statusCode === 403)) {
+ throw error;
+ }
+ const alertError = error instanceof Error ? error : new Error('Daytona sandbox create failed', { cause: error });
+ this.logger.error('Daytona sandbox create failed', {
+ snapshot_ref: env.snapshot_ref,
+ ...extractErrorLogFields(alertError),
+ });
+ this.onCriticalAlert?.(alertError);
+ throw error;
+ }
+ }
+
private async getOrCreateSandbox(
sandboxId?: string,
environment?: DaytonaSandboxEnvironment,
@@ -336,7 +374,7 @@ export class DaytonaSandboxProvider implements SandboxProvider<DaytonaSandboxEnv
const sandbox = sandboxId
? await this.restoreExistingSandbox(sandboxId)
- : await this.daytona.create(this.buildCreateParams(environment));
+ : await this.createFreshSandbox(environment);
const entry = { sandbox, defaultTimeoutMs: this.timeoutMs };
DaytonaSandboxProvider.cachedSandboxes.set(sandbox.name, entry);
diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.ts
index 88b3f8d13..cbf74bac4 100644
--- a/packages/trueforge/src/sandbox/providerUtils.ts
+++ b/packages/trueforge/src/sandbox/providerUtils.ts
@@ -16,6 +16,7 @@ import {
type SandboxStatus,
type StoredSandboxProviderManifest,
} from '../schemas/sandboxProvider';
+import { captureCriticalException } from '../sentry';
/** Bound Daytona RPCs so API requests and controller ticks cannot hang indefinitely. */
export const DAYTONA_RPC_TIMEOUT_MS = 5_000;
@@ -103,6 +104,11 @@ export function toDaytonaSandboxProvider({
tenantName: tenant_id,
fileMaxBytesForDownload: configuration.SANDBOX_FILE_MAX_BYTES_FOR_DOWNLOAD,
logger,
+ onCriticalAlert: error => {
+ captureCriticalException(error, {
+ tags: { module: 'DaytonaProvider', operation: 'sandboxCreate', priority: 'P1' },
+ });
+ },
});
}