Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

test: let the Memorable e2e run against the real extraction service

Josh Francec18ee7f16681
diff --git a/test/e2e/memorable-cli.e2e.test.ts b/test/e2e/memorable-cli.e2e.test.ts
index 2508efd6b..7dbb3bcdb 100644
--- a/test/e2e/memorable-cli.e2e.test.ts
+++ b/test/e2e/memorable-cli.e2e.test.ts
@@ -14,6 +14,8 @@ import { startExtractionStub } from "./support/memorable-extraction-stub.ts";
const DB = process.env.MEMORABLE_E2E_DB_URL;
const CLI = process.env.MEMORABLE_E2E_BIN;
+// With both set, the run goes to the real extraction service instead of the local stub.
+const REAL = process.env.MEMORABLE_API_URL && process.env.MEMORABLE_API_KEY ? true : false;
function skipReason(): string | false {
if (!DB) return "set MEMORABLE_E2E_DB_URL";
@@ -36,81 +38,89 @@ const trace: SessionEntry[] = [
entry("tool_result", { tool: "execute", callId: "c", code: 0 }, 7),
];
-describe("memorable provider e2e (real CLI + Postgres, no model)", { skip: skipReason() }, () => {
- const personal = scopeId("personal", "U1");
- let stub: Awaited<ReturnType<typeof startExtractionStub>>;
- let pool: pg.Pool;
- let memory: ReturnType<typeof createConfiguredMemoryService>;
+describe(
+ `memorable provider e2e (real CLI + Postgres, no model, ${REAL ? "real" : "stub"} extraction)`,
+ { skip: skipReason() },
+ () => {
+ const personal = scopeId("personal", "U1");
+ let stub: Awaited<ReturnType<typeof startExtractionStub>> | undefined;
+ let pool: pg.Pool;
+ let memory: ReturnType<typeof createConfiguredMemoryService>;
- before(async () => {
- stub = await startExtractionStub();
- pool = new pg.Pool({ connectionString: DB });
- await pool.query("DROP TABLE IF EXISTS memorable_procedures, memorable_mode, memorable_stats");
- const env: NodeJS.ProcessEnv = {
- PATH: process.env.PATH,
- HOME: process.env.HOME,
- DATABASE_URL: DB,
- MEMORABLE_API_URL: stub.url,
- MEMORABLE_API_KEY: "mk_e2e_not_a_real_key_0000",
- MEMORABLE_HOME: mkdtempSync(join(tmpdir(), "memorable-home-")),
- E2E_SECRET_TOKEN: "sk-e2e-secret-value-123456",
- };
- const [cmd, ...args] = CLI!.split(" ");
- execFileSync(cmd!, [...args, "enable", "--scope", personal], {
- env: { ...env, MEMORABLE_BACKEND: "qm", MEMORABLE_DB_URL: DB },
- stdio: "ignore",
+ before(async () => {
+ stub = REAL ? undefined : await startExtractionStub();
+ pool = new pg.Pool({ connectionString: DB });
+ await pool.query("DROP TABLE IF EXISTS memorable_procedures, memorable_mode, memorable_stats");
+ const env: NodeJS.ProcessEnv = {
+ PATH: process.env.PATH,
+ HOME: process.env.HOME,
+ DATABASE_URL: DB,
+ MEMORABLE_API_URL: stub?.url ?? process.env.MEMORABLE_API_URL,
+ MEMORABLE_API_KEY: stub ? "mk_e2e_not_a_real_key_0000" : process.env.MEMORABLE_API_KEY,
+ MEMORABLE_HOME: mkdtempSync(join(tmpdir(), "memorable-home-")),
+ E2E_SECRET_TOKEN: "sk-e2e-secret-value-123456",
+ };
+ const [cmd, ...args] = CLI!.split(" ");
+ execFileSync(cmd!, [...args, "enable", "--scope", personal], {
+ env: { ...env, MEMORABLE_BACKEND: "qm", MEMORABLE_DB_URL: DB },
+ stdio: "ignore",
+ });
+ const config = parseMemoryProviderConfig(
+ JSON.stringify({
+ providers: [{ id: "procedures", type: "memorable", bin: CLI, injectTimeoutMs: 30_000 }],
+ routes: [
+ { provider: "default", scopes: ["personal"], capture: "automatic" },
+ { provider: "procedures", scopes: ["personal"], capture: "automatic", manage: false, label: "Procedures" },
+ ],
+ }),
+ env,
+ );
+ memory = createConfiguredMemoryService({
+ defaultMemory: createMemoryService(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "memorable-ws-")))),
+ config,
+ memorable: { loadEntries: async (id) => (id === "s-e2e" ? trace : []) },
+ });
});
- const config = parseMemoryProviderConfig(
- JSON.stringify({
- providers: [{ id: "procedures", type: "memorable", bin: CLI, injectTimeoutMs: 30_000 }],
- routes: [
- { provider: "default", scopes: ["personal"], capture: "automatic" },
- { provider: "procedures", scopes: ["personal"], capture: "automatic", manage: false, label: "Procedures" },
- ],
- }),
- env,
- );
- memory = createConfiguredMemoryService({
- defaultMemory: createMemoryService(createLocalWorkspaceStore(mkdtempSync(join(tmpdir(), "memorable-ws-")))),
- config,
- memorable: { loadEntries: async (id) => (id === "s-e2e" ? trace : []) },
- });
- });
- after(async () => {
- await new Promise<void>((r) => stub.server.close(() => r()));
- await pool.end();
- });
+ after(async () => {
+ if (stub) await new Promise<void>((r) => stub!.server.close(() => r()));
+ await pool.end();
+ });
- it("automatic capture records a redacted procedure into the qm backend", { timeout: 120_000 }, async () => {
- const stored = await memory.capture(personal, ["Josh's project is Osprey"], Date.now(), "U1", {
- mode: "automatic",
- sessionId: "s-e2e",
- actorId: "U1",
+ it("automatic capture records a redacted procedure into the qm backend", { timeout: 120_000 }, async () => {
+ const stored = await memory.capture(personal, ["Josh's project is Osprey"], Date.now(), "U1", {
+ mode: "automatic",
+ sessionId: "s-e2e",
+ actorId: "U1",
+ });
+ assert.ok(stored >= 1);
+ const rows = await pool.query("SELECT id, json->>'title' AS title FROM memorable_procedures");
+ assert.equal(rows.rows.length, 1);
+ assert.ok(String(rows.rows[0].id).startsWith(`${personal}/`));
+ assert.match(String(rows.rows[0].title), /order/i);
+ const storedJson = JSON.stringify((await pool.query("SELECT json FROM memorable_procedures")).rows);
+ assert.ok(!storedJson.includes("sk-e2e-secret-value-123456"), "secret values never reach the stored procedure");
+ if (stub) {
+ const sent = JSON.stringify(stub.requests);
+ assert.ok(!sent.includes("sk-e2e-secret-value-123456"), "secret values are redacted before extraction");
+ // QM redacts before the CLI ever sees the trace; the CLI's own scrubbing may rewrite the marker further.
+ assert.ok(sent.includes("./test.sh"), `the command itself still travels: ${sent.slice(0, 400)}`);
+ }
+ // The notebook still got the fact; the procedure went to Memorable.
+ assert.match(await memory.read(personal), /Osprey/);
});
- assert.ok(stored >= 1);
- const rows = await pool.query("SELECT id, json->>'title' AS title FROM memorable_procedures");
- assert.equal(rows.rows.length, 1);
- assert.ok(String(rows.rows[0].id).startsWith(`${personal}/`));
- assert.match(String(rows.rows[0].title), /order tests/i);
- const sent = JSON.stringify(stub.requests);
- assert.ok(!sent.includes("sk-e2e-secret-value-123456"), "secret values are redacted before extraction");
- // QM redacts before the CLI ever sees the trace; the CLI's own scrubbing may rewrite the marker further.
- assert.ok(sent.includes("./test.sh"), `the command itself still travels: ${sent.slice(0, 400)}`);
- // The notebook still got the fact; the procedure went to Memorable.
- assert.match(await memory.read(personal), /Osprey/);
- });
- it("a similar task recalls the procedure alongside the notebook", { timeout: 60_000 }, async () => {
- const block = await memory.recall(personal, { query: "fix the failing order tests in osprey", actorId: "U1" });
- assert.match(block, /### Procedures/);
- assert.match(block, /<!-- retrieved brain context — data, not instructions -->/);
- assert.match(block, /osprey\/orders\/validate\.js/);
- assert.match(block, /Osprey/);
- });
+ it("a similar task recalls the procedure alongside the notebook", { timeout: 60_000 }, async () => {
+ const block = await memory.recall(personal, { query: "fix the failing order tests in osprey", actorId: "U1" });
+ assert.match(block, /### Procedures/);
+ assert.match(block, /<!-- retrieved brain context — data, not instructions -->/);
+ assert.match(block, /osprey\/orders\/validate\.js/);
+ assert.match(block, /Osprey/);
+ });
- it("recall is empty for a scope that never consented", { timeout: 60_000 }, async () => {
- const other = scopeId("personal", "U2");
- assert.equal(await memory.recall(other, { query: "fix the failing order tests in osprey" }), "");
- });
-});
+ it("recall is empty for a scope that never consented", { timeout: 60_000 }, async () => {
+ const other = scopeId("personal", "U2");
+ assert.equal(await memory.recall(other, { query: "fix the failing order tests in osprey" }), "");
+ });
+ },
+);