The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
feat(server): provision managed sandbox environments from the managed config (#10324)
Devin Foley273315a4d01a
diff --git a/server/src/__tests__/environment-service.test.ts b/server/src/__tests__/environment-service.test.tsindex eff92293fd..4b3ef48810 100644--- a/server/src/__tests__/environment-service.test.ts+++ b/server/src/__tests__/environment-service.test.ts@@ -630,6 +630,147 @@ describeEmbeddedPostgres("environmentService leases", () => {expect((rows[0]?.metadata as Record<string, unknown>)?.managedKubernetesSandbox).toBe(true);});+ it("ensures and refreshes a managed sandbox environment for an arbitrary provider", async () => {+ const created = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona",+ description: "Managed Daytona sandbox environment.",+ provider: "daytona",+ config: { target: "us" },+ });++ expect(created.driver).toBe("sandbox");+ expect(created.name).toBe("Daytona");+ expect(created.config.provider).toBe("daytona");+ expect(created.config.target).toBe("us");+ expect(created.metadata?.managedByPaperclip).toBe(true);+ expect(created.metadata?.managedSandboxProvider).toBe("daytona");++ // Idempotent: a second call refreshes config and name in place, and a+ // description omitted from the spec is cleared, not pinned forever.+ const refreshed = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona (EU)",+ provider: "daytona",+ config: { target: "eu" },+ });+ expect(refreshed.id).toBe(created.id);+ expect(refreshed.name).toBe("Daytona (EU)");+ expect(refreshed.config.target).toBe("eu");+ expect(refreshed.description).toBeNull();++ const rows = await db+ .select()+ .from(environments)+ .where(eq(environments.driver, "sandbox"));+ expect(rows).toHaveLength(1);+ });++ it("adopts the managed slot on a provider switch and drops the stale kubernetes marker", async () => {+ const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" });+ expect(kubernetes.metadata?.managedKubernetesSandbox).toBe(true);++ const daytona = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona",+ provider: "daytona",+ config: { target: "us" },+ });++ expect(daytona.id).toBe(kubernetes.id);+ expect(daytona.name).toBe("Daytona");+ expect(daytona.config.provider).toBe("daytona");+ expect(daytona.config.backend).toBeUndefined();+ expect(daytona.metadata?.managedSandboxProvider).toBe("daytona");+ expect(daytona.metadata?.managedKubernetesSandbox).toBeUndefined();+ expect(await svc.findKubernetesEnvironment()).toBeNull();++ // And back: the kubernetes wrapper re-adopts the same row.+ const restored = await svc.ensureKubernetesEnvironment({ inCluster: true, backend: "job" });+ expect(restored.id).toBe(kubernetes.id);+ expect(restored.metadata?.managedKubernetesSandbox).toBe(true);+ });++ it("archives the managed sandbox row only for its own provider and reactivates on ensure", async () => {+ // Nothing provisioned yet: archiving is a no-op.+ expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull();++ const created = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona",+ provider: "daytona",+ config: { target: "us" },+ });+ expect(created.status).toBe("active");++ // Another provider's unavailability leaves this provider's row alone.+ expect(await svc.archiveManagedSandboxEnvironment({ provider: "kubernetes" })).toBeNull();++ const archived = await svc.archiveManagedSandboxEnvironment({ provider: "daytona" });+ expect(archived?.id).toBe(created.id);+ expect(archived?.status).toBe("archived");++ // Already archived: a repeat call is a no-op.+ expect(await svc.archiveManagedSandboxEnvironment({ provider: "daytona" })).toBeNull();++ // The next healthy boot's ensure re-activates the same row.+ const restored = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona",+ provider: "daytona",+ config: { target: "us" },+ });+ expect(restored.id).toBe(created.id);+ expect(restored.status).toBe("active");+ });++ it("adopts an existing unmanaged sandbox row holding the desired name", async () => {+ const handMade = await svc.create({+ name: "Daytona",+ driver: "sandbox",+ status: "active",+ config: { provider: "daytona", target: "us" },+ });+ expect(handMade.metadata?.managedByPaperclip).toBeUndefined();++ const adopted = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona",+ provider: "daytona",+ config: { target: "eu" },+ });+ expect(adopted.id).toBe(handMade.id);+ expect(adopted.config.target).toBe("eu");+ expect(adopted.metadata?.managedByPaperclip).toBe(true);+ expect(adopted.metadata?.managedSandboxProvider).toBe("daytona");++ const rows = await db+ .select()+ .from(environments)+ .where(eq(environments.driver, "sandbox"));+ expect(rows).toHaveLength(1);+ });++ it("keeps the current name when the desired name belongs to another row", async () => {+ await svc.create({+ name: "Daytona",+ driver: "ssh",+ status: "active",+ config: {+ host: "fixture.example.test",+ port: 22,+ username: "fixture",+ remoteWorkspacePath: "/srv/paperclip",+ },+ });+ const kubernetes = await svc.ensureKubernetesEnvironment({ inCluster: true });++ // The managed slot is adopted, but the rename would collide with the ssh+ // row on environments_name_idx; the ensure keeps the existing name.+ const adopted = await svc.ensureManagedSandboxEnvironment({+ name: "Daytona",+ provider: "daytona",+ config: { target: "us" },+ });+ expect(adopted.id).toBe(kubernetes.id);+ expect(adopted.name).toBe(kubernetes.name);+ expect(adopted.config.provider).toBe("daytona");+ });+it("returns a conflict when creating a second environment with the same name", async () => {await seedEnvironment();diff --git a/server/src/__tests__/managed-config.test.ts b/server/src/__tests__/managed-config.test.tsindex 1717b33288..370999e348 100644--- a/server/src/__tests__/managed-config.test.ts+++ b/server/src/__tests__/managed-config.test.ts@@ -53,6 +53,7 @@ describe("parseManagedConfigEnv", () => {catalogVersion: "2026.720.0",features: { enableApps: false, enablePipelines: true },plugins: { autoInstall: ["daytona", "kubernetes"] },+ environments: [],});});@@ -66,6 +67,7 @@ describe("parseManagedConfigEnv", () => {catalogVersion: "2026.720.0",features: {},plugins: { autoInstall: [] },+ environments: [],});});@@ -223,6 +225,115 @@ describe("parseManagedConfigEnv", () => {});});+describe("parseManagedConfigEnv environments section", () => {+ const entry = (overrides: Record<string, unknown> = {}) => ({+ name: "Daytona",+ provider: "daytona",+ config: { target: "us" },+ ...overrides,+ });++ it("defaults to an empty list when the section is absent (pre-section documents keep booting)", () => {+ const config = parseManagedConfigEnv(envWith(validDoc()));+ expect(config?.environments).toEqual([]);+ });++ it("parses a declared environment and freezes it", () => {+ const config = parseManagedConfigEnv(+ envWith(validDoc({ environments: [entry({ description: "Managed Daytona sandbox." })] })),+ );+ expect(config?.environments).toHaveLength(1);+ const spec = config?.environments[0];+ expect(spec?.name).toBe("Daytona");+ expect(spec?.description).toBe("Managed Daytona sandbox.");+ expect(spec?.provider).toBe("daytona");+ expect(spec?.config).toEqual({ target: "us" });+ expect(Object.isFrozen(config?.environments)).toBe(true);+ expect(Object.isFrozen(spec)).toBe(true);+ expect(Object.isFrozen(spec?.config)).toBe(true);+ });++ it("treats config and description as optional", () => {+ const config = parseManagedConfigEnv(+ envWith(validDoc({ environments: [{ name: "Daytona", provider: "daytona" }] })),+ );+ expect(config?.environments[0]?.config).toEqual({});+ expect(config?.environments[0]?.description).toBeUndefined();+ });++ it("rejects a non-array section and non-object entries", () => {+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: {} }))),+ ).toThrow(/"environments" must be an array/);+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: ["daytona"] }))),+ ).toThrow(/"environments\[0\]" must be an object/);+ });++ it("rejects more than one entry (single managed sandbox slot)", () => {+ expect(() =>+ parseManagedConfigEnv(+ envWith(validDoc({ environments: [entry(), entry({ name: "Other", provider: "kubernetes" })] })),+ ),+ ).toThrow(/at most one entry/);+ });++ it("rejects unknown entry keys", () => {+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ envVars: {} })] }))),+ ).toThrow(/unknown key "envVars"/);+ });++ it("rejects malformed names, descriptions, and providers", () => {+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ name: "" })] }))),+ ).toThrow(/"environments\[0\].name"/);+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ name: " Daytona" })] }))),+ ).toThrow(/"environments\[0\].name"/);+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ description: " " })] }))),+ ).toThrow(/"environments\[0\].description"/);+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ provider: 7 })] }))),+ ).toThrow(/"environments\[0\].provider"/);+ });++ it("rejects a provider that plugins.autoInstall does not provision", () => {+ expect(() =>+ parseManagedConfigEnv(envWith(validDoc({ environments: [entry({ provider: "modal" })] }))),+ ).toThrow(/not in "plugins.autoInstall"/);+ });++ it("rejects a config that sets provider", () => {+ expect(() =>+ parseManagedConfigEnv(+ envWith(validDoc({ environments: [entry({ config: { provider: "daytona" } })] })),+ ),+ ).toThrow(/must not set "provider"/);+ });++ it("rejects secret-bearing config keys at any depth (secrets travel as env vars)", () => {+ expect(() =>+ parseManagedConfigEnv(+ envWith(validDoc({ environments: [entry({ config: { apiKey: "not-a-real-key" } })] })),+ ),+ ).toThrow(/looks secret-bearing/);+ expect(() =>+ parseManagedConfigEnv(+ envWith(validDoc({ environments: [entry({ config: { auth: { accessToken: "t" } } })] })),+ ),+ ).toThrow(/auth.accessToken/);+ expect(() =>+ parseManagedConfigEnv(+ envWith(+ validDoc({ environments: [entry({ config: { adapters: [{ clientSecret: "s" }] } })] }),+ ),+ ),+ ).toThrow(/adapters\[0\].clientSecret/);+ });+});+describe("getManagedInstanceConfig", () => {it("caches by raw env value and reparses when it changes", () => {const raw = validDoc();diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.tsindex f7a277fd71..d309533dde 100644--- a/server/src/__tests__/server-startup-feedback-export.test.ts+++ b/server/src/__tests__/server-startup-feedback-export.test.ts@@ -206,6 +206,7 @@ vi.mock("../services/index.js", () => ({})),feedbackService: feedbackServiceFactoryMock,bootstrapExecutionPolicyFromEnv: vi.fn(async () => null),+ applyManagedEnvironments: vi.fn(async () => null),environmentCustomImageService: environmentCustomImagesServiceFactoryMock,heartbeatService: heartbeatServiceFactoryMock,issueService: vi.fn(() => ({ update: vi.fn(async () => null) })),diff --git a/server/src/app.ts b/server/src/app.tsindex d6f6829684..5eeeb751c0 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -590,7 +590,13 @@ export async function createApp(// swallowed per plugin so the server ALWAYS finishes booting. A degraded// boot (a provider unavailable, some agents cannot run) is strictly// preferable to a crash loop.- void ensureBundledPlugins(+ //+ // The chain is not awaited here (createApp stays fast), but the settled+ // promise is exposed via `app.locals.bundledPluginsStartup` so boot steps+ // that must not outrun plugin availability — managed sandbox environments+ // (`applyManagedEnvironments`) run before the heartbeat resumes queued+ // runs — can sequence on it. It never rejects.+ const bundledPluginsStartup = ensureBundledPlugins(bundledPluginInstalls,{ registry: pluginRegistry, loader, lifecycle, logger },// Managed mode reinstalls soft-uninstalled bundles (the control plane@@ -609,6 +615,7 @@ export async function createApp(}).catch((err) => {logger.error({ err }, "Failed to load ready plugins on startup");});+ app.locals.bundledPluginsStartup = bundledPluginsStartup;let appServicesShutdown = false;const shutdownAppServices = () => {if (appServicesShutdown) return;diff --git a/server/src/index.ts b/server/src/index.tsindex 8161c2cd37..3908d1cfc1 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -41,6 +41,7 @@ import { setupEnvironmentCustomImageTerminalWebSocketServer } from "./realtime/eimport { setupLiveEventsWebSocketServer } from "./realtime/live-events-ws.js";import {feedbackService,+ applyManagedEnvironments,backfillPrincipalAccessCompatibility,backfillLegacyToolOAuthTokens,bootstrapExecutionPolicyFromEnv,@@ -860,6 +861,31 @@ export async function startServer(): Promise<StartedServer> {throw err;}+ // Ensure sandbox environments declared in the managed-config document+ // (`environments` section) before the heartbeat resumes queued runs. The+ // document already parsed fail-closed above; the ensure step itself is+ // fail-safe per entry (a degraded boot beats a fleet-wide crash loop), but+ // a contradictory deployment that also forces PAPERCLIP_EXECUTION_MODE+ // throws here and fails startup. `pluginsReady` sequences the ensure after+ // the bundled-plugin install/load pass so a declared environment never+ // activates before its provider driver is registered; the worker manager+ // additionally gates each entry on a live plugin worker (and archives the+ // row of a provider that did not come up).+ try {+ const bundledPluginsStartup = (app as { locals?: { bundledPluginsStartup?: Promise<unknown> } })+ .locals?.bundledPluginsStartup;+ const managedEnvironmentsResult = await applyManagedEnvironments(db as any, managedConfig, {+ pluginsReady: bundledPluginsStartup,+ workerManager: pluginWorkerManager,+ });+ if (managedEnvironmentsResult) {+ logger.warn(managedEnvironmentsResult, "managed sandbox environments ensured from managed config");+ }+ } catch (err) {+ logger.error({ err }, "failed to apply managed environments from managed config");+ throw err;+ }+let drainHeartbeatRunsForShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<unknown>) | null = null;let prepareHotRestartShutdown: ((signal: "SIGINT" | "SIGTERM") => Promise<{ skipDrain: boolean }>) | null = null;let heartbeatSchedulerStopped = false;diff --git a/server/src/services/environments.ts b/server/src/services/environments.tsindex 6487f7a1b9..d9a4466168 100644--- a/server/src/services/environments.ts+++ b/server/src/services/environments.ts@@ -73,6 +73,26 @@ export interface KubernetesEnvironmentConfigInput {[key: string]: unknown;}+/**+ * Input to `ensureManagedSandboxEnvironment`. Provider-agnostic: `provider`+ * is the sandbox plugin's driver key and is forced into `config.provider`;+ * the rest of `config` is stored verbatim for the plugin to validate at+ * lease time.+ */+export interface ManagedSandboxEnvironmentInput {+ name: string;+ description?: string;+ /** Sandbox provider key (the plugin's driverKey, e.g. "kubernetes", "daytona"). */+ provider: string;+ config?: Record<string, unknown>;+ /**+ * Extra metadata markers stamped on the managed row (e.g. the legacy+ * kubernetes marker `managedKubernetesSandbox` that+ * `findKubernetesEnvironment` keys on).+ */+ extraMetadata?: Record<string, unknown>;+}+function cloneRecord(value: unknown, fallback: Record<string, unknown> | null = null): Record<string, unknown> | null {if (!value || typeof value !== "object" || Array.isArray(value)) return fallback;return { ...(value as Record<string, unknown>) };@@ -181,6 +201,177 @@ function countFromRows(rows: Array<{ count: number | string | null | undefined }}export function environmentService(db: Db) {+ /** The single Paperclip-managed sandbox row (`environments_managed_sandbox_idx`), if present. */+ const findManagedSandboxRow = () =>+ db+ .select()+ .from(environments)+ .where(eq(environments.driver, "sandbox"))+ .then(+ (rows) =>+ rows.find(+ (row) =>+ (row.metadata as Record<string, unknown> | null)?.managedByPaperclip === true,+ ) ?? null,+ );++ /**+ * Idempotently ensure THE Paperclip-managed sandbox environment for this+ * instance, configured for an arbitrary sandbox provider plugin. Mirrors+ * `ensureLocalEnvironment`; the partial unique index+ * `environments_managed_sandbox_idx` enforces at most one managed sandbox+ * row per instance, so this function owns that single slot regardless of+ * provider:+ *+ * - An existing managed row is adopted and refreshed (name, description,+ * config, provider) on every call, so operator/control-plane changes flow+ * via redeploy without recreating the row — including a provider switch,+ * which also drops a stale provider-specific metadata marker.+ * - An existing UNmanaged sandbox row holding the desired name is adopted+ * and stamped as managed, so a row created by hand before the instance+ * became config-managed converges instead of colliding on+ * `environments_name_idx` on every boot.+ */+ const ensureManagedSandboxEnvironment = async (+ input: ManagedSandboxEnvironmentInput,+ ): Promise<Environment> => {+ const desiredConfig: Record<string, unknown> = {+ ...(input.config ?? {}),+ provider: input.provider,+ };+ const desiredMetadata: Record<string, unknown> = {+ managedByPaperclip: true,+ managedSandboxProvider: input.provider,+ ...(input.extraMetadata ?? {}),+ };++ const adopt = async (row: EnvironmentRow): Promise<Environment> => {+ const metadata: Record<string, unknown> = { ...(row.metadata ?? {}), ...desiredMetadata };+ // A provider switch must not leave the previous provider's marker+ // behind (`findKubernetesEnvironment` keys on it).+ if (desiredMetadata[KUBERNETES_MANAGED_MARKER] !== true) {+ delete metadata[KUBERNETES_MANAGED_MARKER];+ }+ const now = new Date();+ const runUpdate = (values: { name?: string }) =>+ db+ .update(environments)+ .set({+ ...values,+ // The row mirrors the managed spec: omitting `description` clears+ // a previously configured one rather than pinning it forever.+ description: input.description ?? null,+ config: desiredConfig,+ metadata,+ status: "active",+ updatedAt: now,+ })+ .where(eq(environments.id, row.id))+ .returning()+ .then((rows) => rows[0] ?? row);+ const updated = await runUpdate({ name: input.name }).catch((error: unknown) => {+ // Another row already holds the desired name; keep the current name+ // rather than failing a boot-time ensure over a display label.+ if (hasConstraintName(error, "environments_name_idx")) {+ return runUpdate({});+ }+ throw error;+ });+ return toEnvironment(updated);+ };++ const existing = await findManagedSandboxRow();+ if (existing) return adopt(existing);++ // The partial unique index `environments_managed_sandbox_idx` enforces+ // "at most one Paperclip-managed sandbox row per instance" at the DB+ // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent+ // callers can race the INSERT; losers re-read the surviving row.+ const now = new Date();+ const inserted = await db+ .insert(environments)+ .values({+ name: input.name,+ description: input.description ?? null,+ driver: "sandbox",+ status: "active",+ config: desiredConfig,+ envVars: {},+ metadata: desiredMetadata,+ createdAt: now,+ updatedAt: now,+ })+ .onConflictDoNothing({+ target: [environments.driver],+ where:+ sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`,+ })+ .returning()+ .then((rows) => rows[0] ?? null)+ .catch((error) => {+ if (+ hasConstraintName(error, "environments_name_idx")+ || hasConstraintName(error, "environments_managed_sandbox_idx")+ ) {+ return null;+ }+ throw error;+ });+ if (inserted) return toEnvironment(inserted);++ // Either a concurrent caller won the managed slot, or an unmanaged row+ // holds the desired name. Adopt whichever exists.+ const winner = await findManagedSandboxRow();+ if (winner) return adopt(winner);+ const sameName = await db+ .select()+ .from(environments)+ .where(eq(environments.name, input.name))+ .then((rows) => rows[0] ?? null);+ if (sameName) {+ if (sameName.driver !== "sandbox") {+ throw new Error(+ `Failed to ensure managed sandbox environment: environment "${input.name}" already exists with driver "${sameName.driver}"`,+ );+ }+ return adopt(sameName);+ }+ throw new Error("Failed to ensure managed sandbox environment");+ };++ /**+ * Archive the Paperclip-managed sandbox row when its provider became+ * unavailable (plugin missing, not ready, or its worker not running), so+ * run scheduling stops selecting an environment whose lease acquisition+ * cannot succeed (`resolveEnvironment` rejects non-active rows).+ *+ * Scoped to the row provisioned for the SAME provider: a row that a+ * provider switch left on a different provider is not touched (the ensure+ * path adopts it once the new provider is healthy). Reactivation is+ * automatic — the next successful `ensureManagedSandboxEnvironment` stamps+ * the row `active` again.+ *+ * Returns the archived environment, or null when there is no active+ * managed row for this provider.+ */+ const archiveManagedSandboxEnvironment = async (+ input: { provider: string },+ ): Promise<Environment | null> => {+ const existing = await findManagedSandboxRow();+ if (!existing || existing.status !== "active") return null;+ const rowProvider = (existing.metadata as Record<string, unknown> | null)+ ?.managedSandboxProvider;+ if (rowProvider !== input.provider) return null;+ const archived = await db+ .update(environments)+ .set({ status: "archived", updatedAt: new Date() })+ // Guarded on status so a concurrent re-activation is not clobbered.+ .where(and(eq(environments.id, existing.id), eq(environments.status, "active")))+ .returning()+ .then((rows) => rows[0] ?? null);+ return archived ? toEnvironment(archived) : null;+ };+return {list: async (companyIdOrFilters?: string | EnvironmentListFilters,@@ -256,15 +447,16 @@ export function environmentService(db: Db) {return toEnvironment(existing);},+ ensureManagedSandboxEnvironment,++ archiveManagedSandboxEnvironment,+/**- * Idempotently ensure a managed Kubernetes sandbox environment exists for a- * instance, configured from instance/operator-supplied config. Mirrors- * `ensureLocalEnvironment`, but there is no DB unique index for sandbox- * drivers, so idempotency is by metadata marker + driver lookup.- *- * The environment is `driver: "sandbox"` with `config.provider:- * "kubernetes"` so it resolves to the first-party Kubernetes sandbox- * provider. On subsequent calls the config is refreshed (so operators can+ * Idempotently ensure a managed Kubernetes sandbox environment exists for+ * an instance, configured from instance/operator-supplied config. A thin+ * wrapper over `ensureManagedSandboxEnvironment` that pins the provider to+ * "kubernetes" and stamps the legacy marker `findKubernetesEnvironment`+ * keys on. On subsequent calls the config is refreshed (so operators can* update egress/runtimeClass via gitops without recreating the row).*/ensureKubernetesEnvironment: async (@@ -272,94 +464,13 @@ export function environmentService(db: Db) {maybeConfig?: KubernetesEnvironmentConfigInput,): Promise<Environment> => {const config = resolveKubernetesConfig(companyIdOrConfig, maybeConfig);- const desiredConfig: Record<string, unknown> = {- ...config,+ return ensureManagedSandboxEnvironment({+ name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME,+ description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION,provider: KUBERNETES_PROVIDER_KEY,- };- const desiredMetadata: Record<string, unknown> = {- managedByPaperclip: true,- [KUBERNETES_MANAGED_MARKER]: true,- };-- const existing = await db- .select()- .from(environments)- .where(eq(environments.driver, "sandbox"))- .then((rows) =>- rows.find(- (row) =>- (row.metadata as Record<string, unknown> | null)?.[KUBERNETES_MANAGED_MARKER] === true,- ) ?? null,- );-- const now = new Date();- if (existing) {- const updated = await db- .update(environments)- .set({- config: desiredConfig,- metadata: { ...(existing.metadata ?? {}), ...desiredMetadata },- status: "active",- updatedAt: now,- })- .where(eq(environments.id, existing.id))- .returning()- .then((rows) => rows[0] ?? existing);- return toEnvironment(updated);- }-- // The partial unique index `environments_managed_sandbox_idx` enforces- // "at most one Paperclip-managed sandbox row per instance" at the DB- // level. Use ON CONFLICT DO NOTHING keyed on that index so concurrent- // callers can race the INSERT; losers re-read the surviving row.- const inserted = await db- .insert(environments)- .values({- name: DEFAULT_KUBERNETES_ENVIRONMENT_NAME,- description: DEFAULT_KUBERNETES_ENVIRONMENT_DESCRIPTION,- driver: "sandbox",- status: "active",- config: desiredConfig,- envVars: {},- metadata: desiredMetadata,- createdAt: now,- updatedAt: now,- })- .onConflictDoNothing({- target: [environments.driver],- where:- sql`${environments.driver} = 'sandbox' AND (${environments.metadata} ->> 'managedByPaperclip')::boolean = true`,- })- .returning()- .then((rows) => rows[0] ?? null)- .catch((error) => {- if (- hasConstraintName(error, "environments_name_idx")- || hasConstraintName(error, "environments_managed_sandbox_idx")- ) {- return null;- }- throw error;- });- if (inserted) return toEnvironment(inserted);-- const winner = await db- .select()- .from(environments)- .where(eq(environments.driver, "sandbox"))- .then(- (rows) =>- rows.find(- (candidate) =>- (candidate.metadata as Record<string, unknown> | null)?.[- KUBERNETES_MANAGED_MARKER- ] === true,- ) ?? null,- );- if (!winner) {- throw new Error("Failed to ensure kubernetes environment");- }- return toEnvironment(winner);+ config,+ extraMetadata: { [KUBERNETES_MANAGED_MARKER]: true },+ });},/**diff --git a/server/src/services/index.ts b/server/src/services/index.tsindex 5f05bcf00d..08e0386c6d 100644--- a/server/src/services/index.ts+++ b/server/src/services/index.ts@@ -116,9 +116,11 @@ export {managedFeatureKeySet,parseManagedConfigEnv,MANAGED_CONFIG_ENV_KEY,+ type ManagedEnvironmentSpec,type ManagedInstanceConfig,} from "./managed-config.js";export { bootstrapExecutionPolicyFromEnv } from "./execution-policy-bootstrap.js";+export { applyManagedEnvironments } from "./managed-environments.js";export { cloudUpstreamService, reconcileCloudUpstreamRunsOnStartup } from "./cloud-upstreams.js";export { companyPortabilityService } from "./company-portability.js";export { teamsCatalogService } from "./teams-catalog.js";diff --git a/server/src/services/managed-config.ts b/server/src/services/managed-config.tsindex 87c37b5332..b723930201 100644--- a/server/src/services/managed-config.ts+++ b/server/src/services/managed-config.ts@@ -9,7 +9,10 @@* "mode": "cloud",* "catalogVersion": "2026.720.0",* "features": { "<feature-key>": true | false, ... },- * "plugins": { "autoInstall": ["daytona", "kubernetes"] }+ * "plugins": { "autoInstall": ["daytona", "kubernetes"] },+ * "environments": [+ * { "name": "Daytona", "provider": "daytona", "config": { "target": "us" } }+ * ]* }** Parsing follows the `execution-policy-bootstrap.ts` doctrine: a pure@@ -47,6 +50,52 @@ export interface ManagedInstanceConfig {catalogVersion: string;features: Readonly<Partial<Record<ManagedExperimentalFeatureKey, boolean>>>;plugins: { readonly autoInstall: readonly string[] };+ /** Sandbox environments the control plane provisions at boot (empty when the section is absent). */+ environments: readonly ManagedEnvironmentSpec[];+}++/**+ * One declared managed sandbox environment. Provider-agnostic: `provider` is+ * the sandbox plugin's driver key (== its `plugins.autoInstall` catalog key),+ * and `config` is stored verbatim in the environment row for the plugin to+ * validate at lease time — the same split `ensureKubernetesEnvironment` has+ * always used, generalized to any bundled sandbox provider.+ */+export interface ManagedEnvironmentSpec {+ /** Display name of the instance-level environment row (unique per instance). */+ name: string;+ description?: string;+ /** Sandbox provider key (the plugin's driverKey, e.g. "daytona"). */+ provider: string;+ /** Provider config stored in `environment.config`; never carries secrets. */+ config: Readonly<Record<string, unknown>>;+}++/**+ * Managed-config documents deliver NO secrets, ever. Provider credentials+ * reach a managed instance as process environment variables (every bundled+ * sandbox provider falls back to its env var when `config` omits the key,+ * e.g. `DAYTONA_API_KEY`), so any secret-looking config key in the document+ * is a misrouted credential and fails startup.+ */+const SECRET_LIKE_CONFIG_KEY_PATTERN = /(api[-_]?key|token|secret|password|credential)/i;++function findSecretLikeConfigKey(value: Record<string, unknown>, path: string): string | null {+ for (const [key, child] of Object.entries(value)) {+ const childPath = path.length > 0 ? `${path}.${key}` : key;+ if (SECRET_LIKE_CONFIG_KEY_PATTERN.test(key)) return childPath;+ if (isPlainObject(child)) {+ const nested = findSecretLikeConfigKey(child, childPath);+ if (nested) return nested;+ } else if (Array.isArray(child)) {+ for (const [index, element] of child.entries()) {+ if (!isPlainObject(element)) continue;+ const nested = findSecretLikeConfigKey(element, `${childPath}[${index}]`);+ if (nested) return nested;+ }+ }+ }+ return null;}let cachedFeatureKeys: ReadonlySet<string> | null = null;@@ -106,10 +155,10 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceConfail(`must be a JSON object (got ${describeJsonValue(doc)})`);}- const allowedTopLevelKeys = new Set(["v", "mode", "catalogVersion", "features", "plugins"]);+ const allowedTopLevelKeys = new Set(["v", "mode", "catalogVersion", "features", "plugins", "environments"]);for (const key of Object.keys(doc)) {if (!allowedTopLevelKeys.has(key)) {- fail(`has unknown top-level key "${key}" (allowed: v, mode, catalogVersion, features, plugins)`);+ fail(`has unknown top-level key "${key}" (allowed: v, mode, catalogVersion, features, plugins, environments)`);}}@@ -195,12 +244,93 @@ export function parseManagedConfigEnv(env: ManagedConfigEnv): ManagedInstanceConautoInstall.push(entry);}+ // `environments` is OPTIONAL, unlike `features` and `plugins`: documents+ // delivered before the section existed must keep booting newer builds (a+ // fleet image roll cannot be lockstepped with a config re-delivery), and+ // absence is not a dropped security control — it simply declares no managed+ // environments. When present, the section is validated fail-closed like+ // everything else.+ const environmentSpecs: ManagedEnvironmentSpec[] = [];+ if (doc.environments !== undefined) {+ if (!Array.isArray(doc.environments)) {+ fail(`"environments" must be an array of environment objects (got ${describeJsonValue(doc.environments)})`);+ }+ // The DB enforces at most ONE Paperclip-managed sandbox row per instance+ // (partial unique index `environments_managed_sandbox_idx`); every entry+ // here provisions that row, so a longer list can never be satisfied.+ if (doc.environments.length > 1) {+ fail(+ `"environments" supports at most one entry: each entry provisions the single Paperclip-managed sandbox environment (DB invariant environments_managed_sandbox_idx)`,+ );+ }+ for (const [index, entry] of doc.environments.entries()) {+ if (!isPlainObject(entry)) {+ fail(`"environments[${index}]" must be an object (got ${describeJsonValue(entry)})`);+ }+ const allowedEntryKeys = new Set(["name", "description", "provider", "config"]);+ for (const key of Object.keys(entry)) {+ if (!allowedEntryKeys.has(key)) {+ fail(`"environments[${index}]" has unknown key "${key}" (allowed: name, description, provider, config)`);+ }+ }+ if (typeof entry.name !== "string" || entry.name.length === 0 || entry.name.trim() !== entry.name) {+ fail(+ `"environments[${index}].name" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.name)})`,+ );+ }+ if (+ entry.description !== undefined+ && (typeof entry.description !== "string" || entry.description.trim().length === 0)+ ) {+ fail(`"environments[${index}].description" must be a non-empty string when present (got ${describeJsonValue(entry.description)})`);+ }+ if (typeof entry.provider !== "string" || entry.provider.length === 0 || entry.provider.trim() !== entry.provider) {+ fail(+ `"environments[${index}].provider" must be a non-empty string without surrounding whitespace (got ${describeJsonValue(entry.provider)})`,+ );+ }+ // Coherence: on a managed instance the control plane is the only plugin+ // install path, so an environment whose provider plugin is not+ // auto-installed could never serve a lease. Catch the skew at parse time.+ if (!autoInstall.includes(entry.provider)) {+ fail(+ `"environments[${index}].provider" is "${entry.provider}", which is not in "plugins.autoInstall"; a managed environment requires its provider plugin to be provisioned`,+ );+ }+ const config: Record<string, unknown> = {};+ if (entry.config !== undefined) {+ if (!isPlainObject(entry.config)) {+ fail(`"environments[${index}].config" must be an object (got ${describeJsonValue(entry.config)})`);+ }+ if (entry.config.provider !== undefined) {+ fail(`"environments[${index}].config" must not set "provider"; it is forced from the entry's provider key`);+ }+ const secretLikeKey = findSecretLikeConfigKey(entry.config, "");+ if (secretLikeKey) {+ fail(+ `"environments[${index}].config" key "${secretLikeKey}" looks secret-bearing; credentials are delivered to managed instances as process environment variables (the provider's documented env fallback), never in the managed-config document`,+ );+ }+ Object.assign(config, entry.config);+ }+ environmentSpecs.push(+ Object.freeze({+ name: entry.name,+ ...(entry.description !== undefined ? { description: entry.description } : {}),+ provider: entry.provider,+ config: Object.freeze(config),+ }) as ManagedEnvironmentSpec,+ );+ }+ }+return Object.freeze({v: SUPPORTED_MANAGED_CONFIG_VERSION,mode: "cloud",catalogVersion: doc.catalogVersion,features: Object.freeze(features),plugins: Object.freeze({ autoInstall: Object.freeze(autoInstall) }),+ environments: Object.freeze(environmentSpecs),}) as ManagedInstanceConfig;}diff --git a/server/src/services/managed-environments.test.ts b/server/src/services/managed-environments.test.tsnew file mode 100644index 0000000000..4857c26a8b--- /dev/null+++ b/server/src/services/managed-environments.test.ts@@ -0,0 +1,415 @@+import { describe, expect, it, vi } from "vitest";+import type { Db } from "@paperclipai/db";+import { MANAGED_CONFIG_ENV_KEY, parseManagedConfigEnv } from "./managed-config.js";+import {+ applyManagedEnvironments,+ type ApplyManagedEnvironmentsOptions,+} from "./managed-environments.js";++const noDb = null as unknown as Db;++function parsedConfig(overrides: Record<string, unknown> = {}) {+ const config = parseManagedConfigEnv({+ [MANAGED_CONFIG_ENV_KEY]: JSON.stringify({+ v: 1,+ mode: "cloud",+ catalogVersion: "2026.720.0",+ features: {},+ plugins: { autoInstall: ["daytona"] },+ ...overrides,+ }),+ });+ if (!config) throw new Error("expected a parsed managed config");+ return config;+}++function environmentRow(overrides: Record<string, unknown> = {}) {+ const now = new Date();+ return {+ id: "env-1",+ name: "Daytona",+ description: null,+ driver: "sandbox",+ status: "active",+ config: {},+ envVars: {},+ metadata: {},+ createdAt: now,+ updatedAt: now,+ ...overrides,+ };+}++function readyDriverResolver(status = "ready") {+ return vi.fn(async () => ({+ plugin: { id: "plugin-1", pluginKey: "sandbox-providers/daytona", status },+ }));+}++function runningWorkerManager(running = true) {+ return { isRunning: vi.fn(() => running), getWorker: vi.fn(() => undefined) };+}++type WorkerManagerSeam = NonNullable<ApplyManagedEnvironmentsOptions["workerManager"]>;+type RecoveryHandle = NonNullable<ReturnType<WorkerManagerSeam["getWorker"]>>;++/** A worker handle fake that records `ready` listeners so tests can fire them. */+function recoveryHandle() {+ const readyListeners: Array<(payload: { pluginId: string }) => void> = [];+ const handle: RecoveryHandle = {+ on: vi.fn((_event, listener) => {+ readyListeners.push(listener);+ }),+ off: vi.fn((_event, listener) => {+ const at = readyListeners.indexOf(listener);+ if (at !== -1) readyListeners.splice(at, 1);+ }),+ };+ return {+ handle,+ fireReady: () => {+ for (const listener of [...readyListeners]) listener({ pluginId: "plugin-1" });+ },+ };+}++/** Lets the fire-and-forget recovery ensure settle. */+function tick() {+ return new Promise((resolve) => setImmediate(resolve));+}++type EnvironmentsSeam = NonNullable<ApplyManagedEnvironmentsOptions["environments"]>;++function environmentsSeam(overrides: Partial<EnvironmentsSeam> = {}): EnvironmentsSeam {+ return {+ ensureManagedSandboxEnvironment:+ overrides.ensureManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(environmentRow()),+ archiveManagedSandboxEnvironment:+ overrides.archiveManagedSandboxEnvironment ?? vi.fn().mockResolvedValue(null),+ };+}++describe("applyManagedEnvironments", () => {+ it("no-ops for self-hosted instances and for documents without environments", async () => {+ expect(await applyManagedEnvironments(noDb, null)).toBeNull();+ expect(await applyManagedEnvironments(noDb, parsedConfig())).toBeNull();+ });++ it("refuses startup when a forced execution mode also claims the managed sandbox slot", async () => {+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });+ await expect(+ applyManagedEnvironments(noDb, config, {+ env: { PAPERCLIP_EXECUTION_MODE: "kubernetes" },+ }),+ ).rejects.toThrow(/mutually exclusive/);+ });++ it("ensures each declared environment through the provider-agnostic service call", async () => {+ const ensureManagedSandboxEnvironment = vi+ .fn()+ .mockResolvedValue(environmentRow());+ const config = parsedConfig({+ environments: [+ {+ name: "Daytona",+ description: "Managed Daytona sandbox.",+ provider: "daytona",+ config: { target: "us" },+ },+ ],+ });++ const resolveSandboxProviderDriver = readyDriverResolver();+ const workerManager = runningWorkerManager();+ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager,+ environments: environmentsSeam({ ensureManagedSandboxEnvironment }),+ resolveSandboxProviderDriver,+ });++ expect(result).toEqual({ ensured: 1, failed: 0 });+ expect(resolveSandboxProviderDriver).toHaveBeenCalledWith({ db: noDb, driverKey: "daytona" });+ expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1");+ expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);+ expect(ensureManagedSandboxEnvironment).toHaveBeenCalledWith({+ name: "Daytona",+ description: "Managed Daytona sandbox.",+ provider: "daytona",+ config: { target: "us" },+ });+ // The frozen parsed config must not leak into the service (the row's+ // config is mutated downstream when the provider key is forced in).+ const passedConfig = ensureManagedSandboxEnvironment.mock.calls[0]?.[0]?.config;+ expect(Object.isFrozen(passedConfig)).toBe(false);+ });++ it("waits for the bundled-plugin startup pass before ensuring anything", async () => {+ const ensureManagedSandboxEnvironment = vi+ .fn()+ .mockResolvedValue(environmentRow());+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ let releasePlugins!: () => void;+ const pluginsReady = new Promise<void>((resolve) => {+ releasePlugins = resolve;+ });++ const pending = applyManagedEnvironments(noDb, config, {+ env: {},+ pluginsReady,+ workerManager: runningWorkerManager(),+ environments: environmentsSeam({ ensureManagedSandboxEnvironment }),+ resolveSandboxProviderDriver: readyDriverResolver(),+ });++ // Give the ensure every chance to (incorrectly) run early.+ await new Promise((resolve) => setImmediate(resolve));+ expect(ensureManagedSandboxEnvironment).not.toHaveBeenCalled();++ releasePlugins();+ expect(await pending).toEqual({ ensured: 1, failed: 0 });+ expect(ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);+ });++ it("skips an entry whose provider plugin is missing and archives its stale row", async () => {+ const environments = environmentsSeam({+ archiveManagedSandboxEnvironment: vi.fn().mockResolvedValue(environmentRow({ status: "archived" })),+ });+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager: runningWorkerManager(),+ environments,+ resolveSandboxProviderDriver: vi.fn(async () => null),+ });++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({+ provider: "daytona",+ });+ });++ it("skips (and counts failed) an entry whose provider plugin is not ready", async () => {+ const environments = environmentsSeam();+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager: runningWorkerManager(),+ environments,+ resolveSandboxProviderDriver: readyDriverResolver("disabled"),+ });++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({+ provider: "daytona",+ });+ });++ it("skips an entry whose plugin record is ready but whose worker is not running", async () => {+ const environments = environmentsSeam();+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const workerManager = runningWorkerManager(false);+ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager,+ environments,+ resolveSandboxProviderDriver: readyDriverResolver(),+ });++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(workerManager.isRunning).toHaveBeenCalledWith("plugin-1");+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({+ provider: "daytona",+ });+ // No registered handle → no respawn is coming; degraded until next boot.+ expect(workerManager.getWorker).toHaveBeenCalledWith("plugin-1");+ });++ it("reactivates the archived environment once when the provider worker recovers", async () => {+ const environments = environmentsSeam();+ const config = parsedConfig({+ environments: [+ { name: "Daytona", provider: "daytona", config: { target: "us" } },+ ],+ });++ const { handle, fireReady } = recoveryHandle();+ const workerManager = {+ isRunning: vi.fn(() => false),+ getWorker: vi.fn(() => handle),+ };+ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager,+ environments,+ resolveSandboxProviderDriver: readyDriverResolver(),+ });++ // The boot pass itself stays degraded: archived, counted failed.+ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledWith({+ provider: "daytona",+ });+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();++ // Worker manager respawns the worker → the handle re-emits `ready`.+ fireReady();+ fireReady();+ await tick();++ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledWith({+ name: "Daytona",+ description: undefined,+ provider: "daytona",+ config: { target: "us" },+ });+ expect(handle.off).toHaveBeenCalledTimes(1);+ });++ it("re-checks isRunning after subscribing so a recovery during the archive is not missed", async () => {+ const environments = environmentsSeam();+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const { handle } = recoveryHandle();+ // Down at the gate check, already back up by the time the listener is+ // registered — the `ready` event has been missed.+ const workerManager = {+ isRunning: vi+ .fn()+ .mockReturnValueOnce(false)+ .mockReturnValue(true),+ getWorker: vi.fn(() => handle),+ };+ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager,+ environments,+ resolveSandboxProviderDriver: readyDriverResolver(),+ });+ await tick();++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);+ });++ it("does not subscribe for recovery when the plugin record is missing or not ready", async () => {+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ for (const resolveSandboxProviderDriver of [+ vi.fn(async () => null),+ readyDriverResolver("disabled"),+ ]) {+ const workerManager = runningWorkerManager(false);+ await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager,+ environments: environmentsSeam(),+ resolveSandboxProviderDriver,+ });+ expect(workerManager.getWorker).not.toHaveBeenCalled();+ }+ });++ it("logs, not throws, when the recovery re-ensure fails", async () => {+ const environments = environmentsSeam({+ ensureManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")),+ });+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const { handle, fireReady } = recoveryHandle();+ const workerManager = {+ isRunning: vi.fn(() => false),+ getWorker: vi.fn(() => handle),+ };+ await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager,+ environments,+ resolveSandboxProviderDriver: readyDriverResolver(),+ });++ fireReady();+ await tick();+ // The rejection is swallowed into a log line; reaching this point without+ // an unhandled rejection is the assertion.+ expect(environments.ensureManagedSandboxEnvironment).toHaveBeenCalledTimes(1);+ });++ it("fails closed when no worker manager is provided", async () => {+ const environments = environmentsSeam();+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ environments,+ resolveSandboxProviderDriver: readyDriverResolver(),+ });++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.ensureManagedSandboxEnvironment).not.toHaveBeenCalled();+ });++ it("counts a skipped entry once even when archiving its stale row fails", async () => {+ const environments = environmentsSeam({+ archiveManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")),+ });+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager: runningWorkerManager(),+ environments,+ resolveSandboxProviderDriver: vi.fn(async () => null),+ });++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.archiveManagedSandboxEnvironment).toHaveBeenCalledTimes(1);+ });++ it("is fail-safe per entry: an ensure failure is counted, not thrown", async () => {+ const environments = environmentsSeam({+ ensureManagedSandboxEnvironment: vi.fn().mockRejectedValue(new Error("db exploded")),+ });+ const config = parsedConfig({+ environments: [{ name: "Daytona", provider: "daytona" }],+ });++ const result = await applyManagedEnvironments(noDb, config, {+ env: {},+ workerManager: runningWorkerManager(),+ environments,+ resolveSandboxProviderDriver: readyDriverResolver(),+ });++ expect(result).toEqual({ ensured: 0, failed: 1 });+ expect(environments.archiveManagedSandboxEnvironment).not.toHaveBeenCalled();+ });+});diff --git a/server/src/services/managed-environments.ts b/server/src/services/managed-environments.tsnew file mode 100644index 0000000000..fbdf2c6985--- /dev/null+++ b/server/src/services/managed-environments.ts@@ -0,0 +1,239 @@+/**+ * Managed-environment bootstrap (harness → app contract).+ *+ * Managed-cloud instances may declare sandbox environments in the+ * `environments` section of `PAPERCLIP_MANAGED_CONFIG` (parsed fail-closed in+ * `managed-config.ts`). On boot, each declared environment is idempotently+ * ensured as the instance-level Paperclip-managed sandbox row via the+ * provider-agnostic `ensureManagedSandboxEnvironment` — the control plane+ * provisions, tenants use, for any bundled sandbox provider plugin.+ *+ * The failure posture mirrors bundled plugin provisioning+ * (`bundled-plugins.ts`), deliberately split:+ *+ * 1. **Validation fails closed at parse time** (`managed-config.ts`): a+ * malformed section refuses startup with a precise error.+ * 2. **The DB ensure step is fail-safe per entry**: an ensure failure is+ * logged and boot continues degraded (environment unavailable) rather+ * than crash-looping a fleet.+ *+ * Ensuring is additionally synchronized with provider availability: the+ * caller's `pluginsReady` promise (the bundled-plugin install/load pass) is+ * awaited first, and an entry whose provider plugin is not installed, `ready`,+ * AND running a live worker (`workerManager.isRunning`; a `ready` record whose+ * activation failed has no worker and cannot serve leases) afterwards is+ * skipped (counted failed) instead of being written as an active row —+ * otherwise the heartbeat would resume queued runs against an environment+ * whose lease acquisition cannot succeed yet. When such an entry was+ * provisioned by an earlier boot, its still-active row is archived+ * (`archiveManagedSandboxEnvironment`) for the same reason. Re-activation+ * happens on the next healthy boot's ensure, or earlier: when the plugin+ * record is `ready` and only the worker is down (a crash in restart-backoff),+ * a one-shot `ready` listener on the worker handle re-runs the ensure as soon+ * as the worker manager respawns the worker, so a transient crash does not+ * leave the environment archived until someone restarts the server.+ *+ * Removing an entry from the document stops future refreshes but never+ * deletes or archives the row — there is intentionally no unprovision path+ * here, matching `plugins.autoInstall` semantics (leases may still reference+ * the row; withdrawal is an explicit operator action). Archiving above is+ * scoped to a declared-but-unavailable provider, not to document removal.+ *+ * Provider credentials are never part of the declared config: every bundled+ * sandbox provider falls back to its documented process environment variable+ * (e.g. `DAYTONA_API_KEY`) when `config` omits the key, so the deployment+ * delivers secrets as env vars and the managed document stays secret-free.+ */++import type { Db } from "@paperclipai/db";+import { logger } from "../middleware/logger.js";+import { environmentService } from "./environments.js";+import type { ManagedInstanceConfig } from "./managed-config.js";+import { parseExecutionPolicyBootstrapEnv } from "./execution-policy-bootstrap.js";+import { resolvePluginSandboxProviderDriverByKey } from "./plugin-environment-driver.js";+import type { PluginWorkerManager } from "./plugin-worker-manager.js";++export interface ApplyManagedEnvironmentsOptions {+ env?: Record<string, string | undefined>;+ /**+ * Resolves when the bundled-plugin startup pass (install + load; the+ * `ensureBundledPlugins` chain in `createApp`) has finished. Awaited before+ * any environment is ensured so an active row never precedes its provider+ * driver. The promise never rejects (the chain catches internally).+ */+ pluginsReady?: Promise<unknown>;+ /**+ * The app's plugin worker manager, consulted per entry so an environment is+ * only ensured while its provider plugin has a live worker (a `ready`+ * registry record whose activation failed cannot serve leases). Fails+ * closed: without a worker manager every entry is treated as unavailable.+ *+ * `getWorker` is `PluginWorkerManager.getWorker` narrowed to the `ready`+ * subscription used for post-archive recovery: a worker in crash-restart+ * backoff re-emits `ready` when the manager respawns it, and the listener+ * re-ensures the archived environment (see `applyManagedEnvironments`).+ */+ workerManager?: Pick<PluginWorkerManager, "isRunning"> & {+ getWorker(pluginId: string):+ | {+ on(event: "ready", listener: (payload: { pluginId: string }) => void): void;+ off(event: "ready", listener: (payload: { pluginId: string }) => void): void;+ }+ | undefined;+ };+ /** Test seam: overrides the environment service built from `db`. */+ environments?: Pick<+ ReturnType<typeof environmentService>,+ "ensureManagedSandboxEnvironment" | "archiveManagedSandboxEnvironment"+ >;+ /** Test seam: overrides the sandbox-provider plugin driver lookup. */+ resolveSandboxProviderDriver?: (input: {+ db: Db;+ driverKey: string;+ }) => Promise<{ plugin: { id: string; pluginKey: string; status: string } } | null>;+}++/**+ * Ensure every environment declared in the managed-config document. Returns+ * null when there is nothing to do (self-hosted, or no `environments`+ * section); otherwise the ensured/failed counts. Idempotent; safe to call on+ * every boot.+ */+export async function applyManagedEnvironments(+ db: Db,+ managedConfig: ManagedInstanceConfig | null,+ opts: ApplyManagedEnvironmentsOptions = {},+): Promise<{ ensured: number; failed: number } | null> {+ if (!managedConfig || managedConfig.environments.length === 0) return null;++ // The forced-execution-mode bootstrap (`PAPERCLIP_EXECUTION_MODE=kubernetes`)+ // and this one both own the single Paperclip-managed sandbox row+ // (`environments_managed_sandbox_idx`). Configuring both is contradictory;+ // refuse startup rather than let bootstrap ordering pick a winner.+ const env = opts.env ?? process.env;+ if (parseExecutionPolicyBootstrapEnv(env)) {+ throw new Error(+ `PAPERCLIP_EXECUTION_MODE and the PAPERCLIP_MANAGED_CONFIG "environments" section are mutually exclusive: both manage the single instance sandbox environment`,+ );+ }++ // The heartbeat resumes queued runs right after this bootstrap step, and+ // lease acquisition fails hard on a provider whose plugin is missing or not+ // ready. Wait for the bundled-plugin startup pass to finish, then refuse to+ // ensure (and in particular to re-activate) a row whose provider driver did+ // not come up — a degraded boot without the row beats an active environment+ // that fails every lease until the plugin recovers.+ await opts.pluginsReady;++ const resolveDriver = opts.resolveSandboxProviderDriver ?? resolvePluginSandboxProviderDriverByKey;+ const environments = opts.environments ?? environmentService(db);++ // Recovery path for a `ready` plugin whose worker was down at check time:+ // that shape is usually a crash in restart-backoff, and the manager's+ // respawn re-emits `ready` on the same handle. A one-shot listener re-runs+ // the idempotent ensure so the recovered provider becomes selectable again+ // without waiting for the next boot (this pass has already archived the+ // row). The post-subscribe `isRunning` re-check closes the race where the+ // worker recovered between the gate check and the subscription. No handle+ // means no respawn is coming (activation never started a worker), so the+ // degraded-until-next-boot posture stands.+ const scheduleRecoveryReactivation = (+ spec: ManagedInstanceConfig["environments"][number],+ pluginId: string,+ ): void => {+ const manager = opts.workerManager;+ const handle = manager?.getWorker(pluginId);+ if (!handle) return;+ let reactivated = false;+ const reactivate = (): void => {+ if (reactivated) return;+ reactivated = true;+ handle.off("ready", reactivate);+ void environments+ .ensureManagedSandboxEnvironment({+ name: spec.name,+ description: spec.description,+ provider: spec.provider,+ config: { ...spec.config },+ })+ .then((environment) => {+ logger.info(+ { environmentId: environment.id, name: spec.name, provider: spec.provider },+ "managed sandbox environment reactivated after provider worker recovery",+ );+ })+ .catch((err: unknown) => {+ logger.error(+ { err, name: spec.name, provider: spec.provider },+ "failed to reactivate managed sandbox environment after provider worker recovery (degraded: environment unavailable until next boot)",+ );+ });+ };+ handle.on("ready", reactivate);+ if (manager?.isRunning(pluginId)) reactivate();+ };++ let ensured = 0;+ let failed = 0;+ for (const spec of managedConfig.environments) {+ try {+ const resolved = await resolveDriver({ db, driverKey: spec.provider });+ // `ready` in the registry is necessary but not sufficient: activation+ // can fail after install leaves the record `ready`, in which case no+ // worker is running and the driver cannot serve leases.+ const workerRunning =+ resolved != null && (opts.workerManager?.isRunning(resolved.plugin.id) ?? false);+ if (!resolved || resolved.plugin.status !== "ready" || !workerRunning) {+ failed += 1;+ // A row provisioned by an earlier boot must not stay active either —+ // archive it so run scheduling stops selecting it. Best-effort: an+ // archive failure is logged, and the entry is already counted failed.+ const archived = await environments+ .archiveManagedSandboxEnvironment({ provider: spec.provider })+ .catch((archiveErr: unknown) => {+ logger.error(+ { err: archiveErr, name: spec.name, provider: spec.provider },+ "failed to archive the managed sandbox environment of an unavailable provider",+ );+ return null;+ });+ logger.error(+ {+ name: spec.name,+ provider: spec.provider,+ pluginKey: resolved?.plugin.pluginKey ?? null,+ pluginStatus: resolved?.plugin.status ?? null,+ workerRunning,+ archivedEnvironmentId: archived?.id ?? null,+ },+ "managed sandbox environment provider plugin is not installed, ready, and running; skipping ensure and archiving any previously provisioned row (degraded: environment unavailable)",+ );+ // Only a `ready` record can recover without another boot: the worker+ // manager restarts crashed workers, but nothing (re)installs a missing+ // or non-ready plugin at runtime.+ if (resolved && resolved.plugin.status === "ready") {+ scheduleRecoveryReactivation(spec, resolved.plugin.id);+ }+ continue;+ }+ const environment = await environments.ensureManagedSandboxEnvironment({+ name: spec.name,+ description: spec.description,+ provider: spec.provider,+ config: { ...spec.config },+ });+ ensured += 1;+ logger.info(+ { environmentId: environment.id, name: environment.name, provider: spec.provider },+ "managed sandbox environment ensured",+ );+ } catch (err) {+ failed += 1;+ logger.error(+ { err, name: spec.name, provider: spec.provider },+ "failed to ensure managed sandbox environment; continuing boot (degraded: environment unavailable)",+ );+ }+ }+ return { ensured, failed };+}