Skip to content

The fastest, cheapest, most efficient open-source agent harness. Run more than 500 agents on a $10 VPS.

fix(scripts): storage-scenarios review fixes

M3gA-Mind4590c7983d9a
diff --git a/scripts/storage-scenarios/README.md b/scripts/storage-scenarios/README.md
index 6a7e15ae56..5aa96f4622 100644
--- a/scripts/storage-scenarios/README.md
+++ b/scripts/storage-scenarios/README.md
@@ -11,8 +11,9 @@ a person to read; **nothing is fixed during a run**.
# every driver (sqlite, then file, then memory)
node scripts/storage-scenarios/run.mjs
-# one driver, some scenarios (by id prefix)
-node scripts/storage-scenarios/run.mjs --driver sqlite --only A,B,C
+# one driver, some scenarios (by id prefix; prerequisites run too, e.g. B, C,
+# H and I bring A, F brings E)
+node scripts/storage-scenarios/run.mjs --driver sqlite --only B,C
# re-render the report of an old run
node scripts/storage-scenarios/run.mjs --grade-only target/storage-scenarios/<run-id>
@@ -124,6 +125,10 @@ when it is not, #8076), then writes the marker.
- Each core gets a **random master key**, passed through the environment only.
It, every token the run mints and every fixture secret are scrubbed from
`core.log` and `rpc.jsonl`.
+- **No backend is reachable**: every config the run writes sets
+ `api_url = "http://127.0.0.1:9"`, so a call the core makes on its own
+ (notification triage, the first use of J's fictional API key) fails on a
+ closed loopback port instead of reaching a network or a model.
- **No background job runs** (`[scheduler_gate] mode = "off"` in every config
the run writes, including each signed-in user's): the legacy fixture holds
cron jobs that are due. Composio is off.
diff --git a/scripts/storage-scenarios/isolated.mjs b/scripts/storage-scenarios/isolated.mjs
index 0cdc879c85..f06403a3f3 100644
--- a/scripts/storage-scenarios/isolated.mjs
+++ b/scripts/storage-scenarios/isolated.mjs
@@ -31,7 +31,9 @@ async function writeUserData(rig, user, tag) {
await rig.tryRpc("openhuman.threads_upsert", { id: d.thread, title: d.marker, created_at: new Date().toISOString() });
await rig.tryRpc("openhuman.threads_message_append", { thread_id: d.thread, message: { id: `${d.thread}-m0`, content: d.marker, type: "text", extraMetadata: {}, sender: "user", createdAt: new Date().toISOString() } });
await rig.tryRpc("openhuman.channel_web_voice_note_store", { note_id: d.note, audio_base64: Buffer.from(d.marker).toString("base64") });
- await rig.tryRpc("openhuman.auth_store_provider_credentials", { provider: d.provider, profile: "default", token: `storscen-token-${randomBytes(8).toString("hex")}` });
+ const token = `storscen-token-${randomBytes(8).toString("hex")}`;
+ rig.addSecret(token);
+ await rig.tryRpc("openhuman.auth_store_provider_credentials", { provider: d.provider, profile: "default", token });
return d;
}
@@ -170,10 +172,13 @@ const legacyImport = {
// The imported domains' own collections in the tenant scope; storage
// bookkeeping (tsd.*, leases, journals) and module scopes move on their own.
const imported = (r) => r.scope === "local" && !/^tsd\.|journal|leases|admission|^storage\./.test(r.coll);
- const delta = second.census.filter(imported).filter((r) => {
- const was = first.census.find((x) => x.scope === r.scope && x.coll === r.coll);
- return !was || was.n !== r.n;
- });
+ // Both directions: a collection emptied by the second boot is absent
+ // from its census and must still count.
+ const count = (census, r) => census.find((x) => x.scope === r.scope && x.coll === r.coll)?.n ?? 0;
+ const delta = [...first.census.filter(imported), ...second.census.filter(imported)]
+ .filter((r, i, all) => all.findIndex((x) => x.scope === r.scope && x.coll === r.coll) === i)
+ .filter((r) => count(first.census, r) !== count(second.census, r))
+ .map((r) => ({ scope: r.scope, coll: r.coll, first: count(first.census, r), second: count(second.census, r) }));
const legacyMarkers = (m) => m.filter((x) => x.startsWith("done/") && x.includes("storage_import/")).sort();
ctx.check({ id: "E-idempotent", ok: delta.length === 0 && json(legacyMarkers(second.markers)) === json(legacyMarkers(first.markers)), expected: "a second boot adds no imported record and no import marker", actual: delta.length ? json(delta).slice(0, 300) : `unchanged; ${legacyMarkers(second.markers).length} markers` });
const other = second.census.filter((r) => !imported(r)).filter((r) => (first.census.find((x) => x.scope === r.scope && x.coll === r.coll)?.n ?? 0) !== r.n);
@@ -184,6 +189,7 @@ const legacyImport = {
const layoutMigration = {
id: "F-layout-migration",
+ needs: ["E-legacy-import"],
title: "Layout v2 opt-in migration: archive, data intact; no master key leaves credentials in place (#8076)",
drivers: ["sqlite"],
async run(ctx) {
diff --git a/scripts/storage-scenarios/run.mjs b/scripts/storage-scenarios/run.mjs
index f2f026e506..37c4ed32fc 100644
--- a/scripts/storage-scenarios/run.mjs
+++ b/scripts/storage-scenarios/run.mjs
@@ -67,6 +67,10 @@ function parseArgs(argv) {
const BASE_CONFIG = [
"schema_version = 13",
+ // Every backend-bound call (notification triage, a fictional API key's
+ // first use) goes to a closed loopback port: the run never reaches a
+ // network or a model.
+ 'api_url = "http://127.0.0.1:9"',
"onboarding_completed = true",
"chat_onboarding_completed = true",
"",
@@ -244,37 +248,45 @@ async function runDriver(driver, { opts, runDir, runId, findings, results }) {
},
});
- await main.start();
- const wanted = (s) => !opts.only.length || opts.only.some((o) => s.id.toUpperCase().startsWith(o));
- const list = [...SCENARIOS, ...ISOLATED].filter((s) => s.drivers.includes(driver) && wanted(s));
- for (const s of list) {
- binding.scenario = s.id;
- const before = findings.checks.length;
- const notesBefore = findings.items.filter((f) => f.severity === "note").length;
- const started = Date.now();
- let error = null;
- console.log(`${driver.padEnd(7)}: ${s.id}`);
- try {
- if (!main.alive) await main.start();
- await s.run(ctx);
- } catch (e) {
- error = scrub(e.message, secrets);
- findings.add({ engine: driver, scenario: s.id, id: "crashed", severity: "high", expected: "the scenario runs to the end", actual: error, evidence: `${driver}/core.log` });
- } finally {
- // Isolated cores never outlive their scenario (F restarts E's rig itself).
- for (const r of isolated.splice(0)) await r.stop().catch(() => {});
+ const all = [...SCENARIOS, ...ISOLATED].filter((s) => s.drivers.includes(driver));
+ const picked = (s) => !opts.only.length || opts.only.some((o) => s.id.toUpperCase().startsWith(o));
+ // A scenario's prerequisites (`needs`) run with it, so `--only B` still
+ // has A's records to check.
+ const needed = new Set(all.filter(picked).flatMap((s) => [s.id, ...(s.needs ?? [])]));
+ const list = all.filter((s) => needed.has(s.id));
+ try {
+ await main.start();
+ for (const s of list) {
+ binding.scenario = s.id;
+ const before = findings.checks.length;
+ const notesBefore = findings.items.filter((f) => f.severity === "note").length;
+ const started = Date.now();
+ let error = null;
+ console.log(`${driver.padEnd(7)}: ${s.id}`);
+ try {
+ if (!main.alive) await main.start();
+ await s.run(ctx);
+ } catch (e) {
+ error = scrub(e.message, secrets);
+ findings.add({ engine: driver, scenario: s.id, id: "crashed", severity: "high", expected: "the scenario runs to the end", actual: error, evidence: `${driver}/core.log` });
+ } finally {
+ // Isolated cores never outlive their scenario (F restarts E's rig itself).
+ for (const r of isolated.splice(0)) await r.stop().catch(() => {});
+ }
+ const checks = findings.checks.slice(before).filter((c) => c.engine === driver);
+ driverResults.scenarios[s.id] = {
+ checks: checks.length,
+ failed: checks.filter((c) => !c.ok).length,
+ notes: findings.items.filter((f) => f.severity === "note").length - notesBefore,
+ ms: Date.now() - started,
+ error,
+ };
}
- const checks = findings.checks.slice(before).filter((c) => c.engine === driver);
- driverResults.scenarios[s.id] = {
- checks: checks.length,
- failed: checks.filter((c) => !c.ok).length,
- notes: findings.items.filter((f) => f.severity === "note").length - notesBefore,
- ms: Date.now() - started,
- error,
- };
+ } finally {
+ // A failed boot or scenario never leaves a core running.
+ await main.stop().catch(() => {});
+ if (ctx.state.legacy?.rig) await ctx.state.legacy.rig.stop().catch(() => {});
}
- await main.stop();
- if (ctx.state.legacy?.rig) await ctx.state.legacy.rig.stop().catch(() => {});
}
function renderReport({ runId, results, findings }) {
diff --git a/scripts/storage-scenarios/scenarios.mjs b/scripts/storage-scenarios/scenarios.mjs
index ddc5086cfa..01185ed659 100644
--- a/scripts/storage-scenarios/scenarios.mjs
+++ b/scripts/storage-scenarios/scenarios.mjs
@@ -254,6 +254,7 @@ const writeRead = {
const durability = {
id: "B-durability",
title: "Restart keeps everything; kill -9 mid-write leaves no torn records",
+ needs: ["A-write-read"],
drivers: ["sqlite", "file"],
async run(ctx) {
await ctx.restart();
@@ -335,6 +336,7 @@ const durability = {
const onDisk = {
id: "C-on-disk",
title: "Records sit in the expected scope and collection; secrets are not plaintext",
+ needs: ["A-write-read"],
drivers: ["sqlite", "file"],
async run(ctx) {
const a = ctx.state.a;
@@ -406,6 +408,7 @@ const onDisk = {
const memoryRestart = {
id: "H-memory-restart",
title: "memory driver: everything gone after a restart, as designed",
+ needs: ["A-write-read"],
drivers: ["memory"],
async run(ctx) {
const a = ctx.state.a;
@@ -421,7 +424,7 @@ const memoryRestart = {
ctx.check({ id: "H-cron-gone", ok: control && !jobs.some((j) => j.id === a.cronId), expected: "the A cron job is gone after a restart", actual: control ? (jobs.some((j) => j.id === a.cronId) ? "still listed" : "gone") : "control failed" });
const msgs = await ctx.tryRpc("openhuman.threads_messages_list", { thread_id: a.threadId });
const n = listOf(msgs.value, "messages").length;
- ctx.check({ id: "H-stream-gone", ok: control && n === 0, expected: "the A thread has no messages after a restart", actual: msgs.ok ? `${n} messages` : `error (${errText(msgs).slice(0, 120)})` });
+ ctx.check({ id: "H-stream-gone", ok: control && msgs.ok && n === 0, expected: "the A thread has no messages after a restart", actual: msgs.ok ? `${n} messages` : `error (${errText(msgs).slice(0, 120)})` });
},
};
@@ -432,6 +435,7 @@ const memoryRestart = {
const concurrency = {
id: "I-concurrency",
title: "Concurrent writers: compare-and-swap has one winner; parallel appends lose nothing",
+ needs: ["A-write-read"],
drivers: ["sqlite"],
async run(ctx) {
if (ctx.has("openhuman.flows_create") && ctx.has("openhuman.flows_update")) {