Multiplayer agent harness for work.
Distinguish not-yet-deployed targets from secret drift in qm doctor (#28)
Regan Bell8da464cbcbe8
diff --git a/cli/src/backends/aws.ts b/cli/src/backends/aws.tsindex b7f87d0c4..9c5db74ba 100644--- a/cli/src/backends/aws.ts+++ b/cli/src/backends/aws.ts@@ -2694,6 +2694,39 @@ export function assertAwsDeploymentStorage(config: QmConfig): void {]);}+export function probeAwsSecretStore(+ secrets: ComputedSecret[],+ read: (name: string) => string,+ checkPublicApiUrl: () => void,+): { values: Map<string, string>; pending: string[]; failures: string[] } {+ const values = new Map<string, string>();+ const pending: string[] = [];+ const failures: string[] = [];+ for (const secret of secrets) {+ const label = secret.required ? `secret ${secret.name}` : `optional secret ${secret.name}`;+ try {+ const value = read(secret.name);+ if (isInvalidSecret(secret.name, value)) throw new Error("missing, placeholder, or insecure value");+ values.set(secret.name, value);+ if (secret.required && secret.name === "PUBLIC_API_URL") checkPublicApiUrl();+ step(secret.required ? `${label}: ok` : `${label}: configured`);+ } catch (error) {+ if (/ResourceNotFoundException/.test(errMessage(error))) {+ if (secret.required) {+ pending.push(secret.name);+ step(`${label}: not pushed yet — run \`qm secrets push\` before the first deploy`);+ } else {+ warn(`${label}: not configured`);+ }+ } else {+ failures.push(`${label}: ${errMessage(error)}`);+ warn(`${label}: failed`);+ }+ }+ }+ return { values, pending, failures };+}+export async function awsDoctor(config: QmConfig, configDir: string): Promise<void> {const { aws } = awsTopology(config, configDir);header(`qm doctor — ${config.orgId} (aws)`);@@ -2903,32 +2936,21 @@ export async function awsDoctor(config: QmConfig, configDir: string): Promise<vo});check("ALB routing", () => assertAwsPublicRouting(config, ecsServices));await checkAsync("public URL DNS and TLS", () => assertAwsPublicNetwork(config));- const runtimeSecrets = new Map<string, string>();- for (const secret of computedSecrets(config)) {- const inspect = (): void => {- const id = `${aws.secretsPrefix}${secret.name}`;- const value = awsText(aws, ["secretsmanager", "get-secret-value", "--secret-id", id, "--query", "SecretString"]);- if (isInvalidSecret(secret.name, value)) throw new Error("missing, placeholder, or insecure value");- runtimeSecrets.set(secret.name, value);- };- if (secret.required)- check(`secret ${secret.name}`, () => {- inspect();- if (secret.name === "PUBLIC_API_URL") assertAwsPublicApiUrl(config);- });- else {- try {- inspect();- step(`optional secret ${secret.name}: configured`);- } catch (error) {- if (/ResourceNotFoundException/.test(errMessage(error))) warn(`optional secret ${secret.name}: not configured`);- else {- failures.push(`optional secret ${secret.name}: ${errMessage(error)}`);- warn(`optional secret ${secret.name}: failed`);- }- }- }- }+ const probe = probeAwsSecretStore(+ computedSecrets(config),+ (name) =>+ awsText(aws, [+ "secretsmanager",+ "get-secret-value",+ "--secret-id",+ `${aws.secretsPrefix}${name}`,+ "--query",+ "SecretString",+ ]),+ () => assertAwsPublicApiUrl(config),+ );+ failures.push(...probe.failures);+ const runtimeSecrets = probe.values;if (failures.length) throw new CliError(`doctor failed:\n${failures.map((failure) => ` - ${failure}`).join("\n")}`);const runtimeNames = ["SLACK_BOT_TOKEN", "SLACK_APP_TOKEN"] as const;const priorRuntime = new Map(runtimeNames.map((name) => [name, process.env[name]]));@@ -2938,7 +2960,7 @@ export async function awsDoctor(config: QmConfig, configDir: string): Promise<voelse delete process.env[name];}try {- await doctorCommon(config, runtimeSecrets, { configDir, requiredSecretValues: true });+ await doctorCommon(config, runtimeSecrets, { configDir, requiredSecretValues: probe.pending.length === 0 });} finally {for (const name of runtimeNames) {const prior = priorRuntime.get(name);diff --git a/cli/src/backends/doctor.ts b/cli/src/backends/doctor.tsindex 7abf0ad23..37d1a204f 100644--- a/cli/src/backends/doctor.ts+++ b/cli/src/backends/doctor.ts@@ -182,7 +182,9 @@ export async function doctorCommon(step("Slack setup: deferred to the admin connector page");} else {warn(- "SLACK_BOT_TOKEN/SLACK_APP_TOKEN values are not available locally — skipping the live Slack check (secret names were verified on the Fly apps)",+ config.target === "aws"+ ? "SLACK_BOT_TOKEN/SLACK_APP_TOKEN are not in the AWS secret store yet — skipping the live Slack check"+ : "SLACK_BOT_TOKEN/SLACK_APP_TOKEN values are not available locally — skipping the live Slack check (secret names were verified on the Fly apps)",);}}diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.tsindex 429b318cf..0940eb03d 100644--- a/cli/src/backends/fly.ts+++ b/cli/src/backends/fly.ts@@ -313,8 +313,11 @@ function writeDerived(ctx: FlyCtx, service: ServiceName): string {return path;}-function secretNames(app: string): Set<string> {- const out = fly(["secrets", "list", "-a", app], { allow: /app not found|Could not find App/i });+const FLY_APP_NOT_FOUND = /app not found|Could not find App/i;++function secretNames(app: string): Set<string> | undefined {+ const out = fly(["secrets", "list", "-a", app], { allow: FLY_APP_NOT_FOUND });+ if (FLY_APP_NOT_FOUND.test(out)) return undefined;return new Set([...out.matchAll(/^\s*\*?\s*([A-Z0-9_]+)\s/gm)].map((m) => m[1]).filter((s): s is string => s !== undefined),);@@ -331,7 +334,7 @@ function ensureObjectStorage(ctx: FlyCtx): void {const required = flyProviderSecrets(ctx.config, "core");if (!required.length) return;const app = `${ctx.appPrefix}-core`;- if (required.every((name) => secretNames(app).has(name))) {+ if (required.every((name) => secretNames(app)?.has(name))) {note(`object storage: credentials already set on ${app}`);return;}@@ -339,7 +342,7 @@ function ensureObjectStorage(ctx: FlyCtx): void {if (!bucket) throw new CliError("Fly S3 storage requires env.core.S3_BUCKET");note(`object storage: creating private Tigris bucket ${bucket}…`);fly(["storage", "create", "--name", bucket, "--app", app, "--org", ctx.flyOrg, "--yes"]);- const missing = required.filter((name) => !secretNames(app).has(name));+ const missing = required.filter((name) => !secretNames(app)?.has(name));if (missing.length) throw new CliError(`Fly storage create did not attach ${missing.join(", ")} to ${app}`);note(`object storage: credentials staged on ${app}`);}@@ -430,7 +433,7 @@ function ensureApp(app: string, flyOrg: string, orgId: string, appPrefix: stringif (!flyOrgApps(flyOrg).has(app)) {throw new CliError(`app ${app} exists outside configured Fly organization ${flyOrg}; choose another appPrefix`);}- if (!secretNames(app).has(marker)) {+ if (!secretNames(app)?.has(marker)) {throw new CliError(`app ${app} already exists but is not marked as owned by this deployment; ` +`choose another appPrefix or explicitly remove the conflicting app`,@@ -448,14 +451,14 @@ function assertOwnedApp(app: string, flyOrg: string, orgId: string, appPrefix: sthrow new CliError(`app ${app} is not present in configured Fly organization ${flyOrg}`);}const marker = flyOwnershipMarker(flyOrg, orgId, appPrefix);- if (!secretNames(app).has(marker)) {+ if (!secretNames(app)?.has(marker)) {throw new CliError(`app ${app} is not marked as owned by deployment ${flyDeploymentId(flyOrg, orgId, appPrefix)}`);}}function ensurePostgres(ctx: FlyCtx): void {const app = `${ctx.appPrefix}-core`;- const hasDatabaseUrl = secretNames(app).has("DATABASE_URL");+ const hasDatabaseUrl = secretNames(app)?.has("DATABASE_URL");if (hasDatabaseUrl) {note(`postgres: DATABASE_URL already set on ${app}`);return;@@ -945,7 +948,7 @@ function runFlyDeploy(args: string[], cwd: string): Promise<void> {function unsetDisabledSecurityScreenToken(config: QmConfig, appPrefix: string): void {if (config.securityScreen) return;const app = `${appPrefix}-core`;- if (!secretNames(app).has("SECURITY_SCREEN_PROXY_TOKEN")) return;+ if (!secretNames(app)?.has("SECURITY_SCREEN_PROXY_TOKEN")) return;fly(["secrets", "unset", "--stage", "-a", app, "SECURITY_SCREEN_PROXY_TOKEN"]);note(`removed the disabled security screen token from ${app}`);}@@ -953,7 +956,7 @@ function unsetDisabledSecurityScreenToken(config: QmConfig, appPrefix: string):function unsetDisabledFlyPublisherToken(config: QmConfig, appPrefix: string): void {if (config.env.core?.DEPLOY_PROVIDER === "fly") return;const app = `${appPrefix}-core`;- if (!secretNames(app).has("FLY_DEPLOY_API_TOKEN")) return;+ if (!secretNames(app)?.has("FLY_DEPLOY_API_TOKEN")) return;fly(["secrets", "unset", "--stage", "-a", app, "FLY_DEPLOY_API_TOKEN"]);note(`removed the disabled Fly app publisher token from ${app}`);}@@ -1052,7 +1055,7 @@ export async function flyUp(config: QmConfig, configDir: string, opts: FlyUpOpts}const gateSecrets = (app: string, header: string, path: string, required: string[], timingKey: string): boolean => {- const existing = timing.time(timingKey, "secret checks", () => secretNames(app));+ const existing = timing.time(timingKey, "secret checks", () => secretNames(app)) ?? new Set<string>();const missing = required.filter((sec) => !existing.has(sec));note(`\n=== ${header} ===`);note(`config: ${path}`);@@ -1133,7 +1136,7 @@ function listRunningApps(const inspect = (name: string): { owned: boolean; reason?: string } => {let markerFailure = "";try {- if (secretNames(name).has(marker)) return { owned: true };+ if (secretNames(name)?.has(marker)) return { owned: true };} catch (error) {markerFailure = errMessage(error);}@@ -1328,7 +1331,7 @@ export function flyPinSandbox(config: QmConfig, image: string, configDir = proce}const pinned: QmConfig = { ...config, sandbox: { ...config.sandbox, image } };const cfgPath = writeDerived(buildCtx(pinned, configDir, {}), "core");- if (secretNames(app).has("FLY_BASE_IMAGE")) {+ if (secretNames(app)?.has("FLY_BASE_IMAGE")) {fly(["secrets", "unset", "--stage", "-a", app, "FLY_BASE_IMAGE"]);note(`removed the stale FLY_BASE_IMAGE secret on ${app}; the derived [env] pin is authoritative`);}@@ -1380,6 +1383,10 @@ export async function flyDoctor(config: QmConfig, configDir: string, envFile?: sfor (const workload of [...runnableServices(config.services), ...pluginNames]) {const app = `${prefix}-${workload}`;const existing = secretNames(app);+ if (!existing) {+ step(`${app}: not created yet — secret checks run after the first \`qm up\``);+ continue;+ }const declared = secretsForService(config, workload, pluginNames);const required = new Set([...declareddiff --git a/cli/test/aws.test.ts b/cli/test/aws.test.tsindex 9142f4caf..8f045e892 100644--- a/cli/test/aws.test.ts+++ b/cli/test/aws.test.ts@@ -20,6 +20,7 @@ import {githubTrustSubject,imageTransferArgs,isPinnedWorkloadImage,+ probeAwsSecretStore,renderTaskDefinition,serviceEnvironment,taskDefinitionChanges,@@ -4236,3 +4237,51 @@ test("AWS renders adopted logGroup and stopTimeout pins; the defaults stay deriv);assert.equal(container.stopTimeout, 120);});++test("AWS doctor classifies an un-pushed secret store as pending, not drift", () => {+ const rnf = new Error(+ "An error occurred (ResourceNotFoundException) when calling the GetSecretValue operation: Secrets Manager can't find the specified secret.",+ );+ const secrets = computedSecrets({+ contract: 1,+ orgId: "acme",+ publicUrl: "https://acme.example.com",+ target: "aws",+ services: ["core"],+ plugins: [],+ skills: [],+ env: {},+ imageOverrides: {},+ });+ const requiredNames = secrets.filter((secret) => secret.required).map((secret) => secret.name);+ assert.ok(requiredNames.length > 0);+ const probe = probeAwsSecretStore(+ secrets,+ () => {+ throw rnf;+ },+ () => assert.fail("PUBLIC_API_URL must not be validated when the store is empty"),+ );+ assert.deepEqual(probe.pending, requiredNames);+ assert.deepEqual(probe.failures, []);+ assert.equal(probe.values.size, 0);+});++test("AWS doctor still fails a pushed secret store holding a placeholder value", () => {+ const probe = probeAwsSecretStore(+ [+ {+ name: "CORE_SIGNING_SECRET",+ services: ["core"],+ description: "",+ required: true,+ managedBy: "operator",+ },+ ],+ () => "replace-me",+ () => {},+ );+ assert.deepEqual(probe.pending, []);+ assert.equal(probe.failures.length, 1);+ assert.match(probe.failures[0]!, /CORE_SIGNING_SECRET: missing, placeholder, or insecure value/);+});diff --git a/cli/test/doctor.test.ts b/cli/test/doctor.test.tsindex e14cc625e..b94130b0d 100644--- a/cli/test/doctor.test.ts+++ b/cli/test/doctor.test.ts@@ -177,6 +177,43 @@ if (app === "acme-core") process.stdout.write("CAPABILITY_SECRET\\nCONNECTOR_SEC}});+test("Fly doctor reports apps that are not created yet as pending, not missing secrets", async () => {+ const dir = mkdtempSync(join(tmpdir(), "qm-fly-doctor-predeploy-"));+ const bin = join(dir, "fake-fly.cjs");+ const prior = process.env.FLY_BIN;+ writeFileSync(+ bin,+ `#!/usr/bin/env node+const args = process.argv.slice(2);+if (args[0] === "secrets" && args[1] === "list") {+ process.stderr.write("Error: app not found");+ process.exit(1);+}+process.exit(0);+`,+ );+ chmodSync(bin, 0o755);+ process.env.FLY_BIN = bin;+ const log = console.log;+ const lines: string[] = [];+ console.log = (...parts: unknown[]): void => void lines.push(parts.join(" "));+ try {+ await assert.doesNotReject(+ flyDoctor({ ...config, target: "fly", appPrefix: "acme", region: "sjc", flyOrg: "personal" }, dir),+ );+ assert.ok(+ lines.some((line) => line.includes("acme-core: not created yet")),+ `printed: ${lines.join(" | ")}`,+ );+ assert.ok(!lines.some((line) => line.includes("missing")), `printed: ${lines.join(" | ")}`);+ } finally {+ console.log = log;+ if (prior === undefined) delete process.env.FLY_BIN;+ else process.env.FLY_BIN = prior;+ rmSync(dir, { recursive: true, force: true });+ }+});+test("doctor reads the deployment's slack-app-manifest.yml scopes, falling back to the template", () => {const dir = mkdtempSync(join(tmpdir(), "qm-doctor-manifest-"));try {