Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

Remove unused Keychain history reads and index audit lookups (#1332)

Regan Bell9745e3425ec8
diff --git a/src/admin/postgres-credential-usage-sink.ts b/src/admin/postgres-credential-usage-sink.ts
index ed93f2574..561e1a92c 100644
--- a/src/admin/postgres-credential-usage-sink.ts
+++ b/src/admin/postgres-credential-usage-sink.ts
@@ -17,7 +17,17 @@ export function createPostgresCredentialUsageSink(connectionString: string): Cre
table: "credential_usage",
columns: COLUMNS,
- schema: { expectedChecksum: "42188c2f50734398496ddd5a47a9ec3c15a507059c713a4a6ced0b0b81d908eb" },
+ schema: {
+ expectedChecksum: "42188c2f50734398496ddd5a47a9ec3c15a507059c713a4a6ced0b0b81d908eb",
+ followUps: [
+ {
+ id: "admin/scoped-events/credential_usage/0002",
+ statements: [
+ "CREATE INDEX CONCURRENTLY IF NOT EXISTS credential_usage_by_slug_ts_id ON credential_usage(slug, ts DESC, id DESC)",
+ ],
+ },
+ ],
+ },
defaultLimit: 5000,
equalityFilters: { scopeId: "scope_label", slug: "slug" },
persistErrorMessage: "[credential-usage] failed to persist broker call:",
diff --git a/src/api/agent-api-catalog.ts b/src/api/agent-api-catalog.ts
index f434eaf2f..1270e9b43 100644
--- a/src/api/agent-api-catalog.ts
+++ b/src/api/agent-api-catalog.ts
@@ -592,8 +592,7 @@ const FAMILIES: AgentApiFamily[] = [
{
method: "GET",
path: "/v1/keychain/overview",
- summary:
- "list this user's credential metadata, grants, pending asks, and recent audited use (never secret values)",
+ summary: "list this user's credential metadata, grants, and pending asks (never secret values)",
},
{ method: "DELETE", path: "/v1/keychain/credentials/:id", summary: "remove a registered login" },
{
diff --git a/src/api/routes/keychain.ts b/src/api/routes/keychain.ts
index 7f19f733b..70d8b049c 100644
--- a/src/api/routes/keychain.ts
+++ b/src/api/routes/keychain.ts
@@ -165,28 +165,11 @@ async function handleKeychain(ctx: ApiCtx): Promise<void> {
const connectorCredentials = (await kc.listConnectorsByOwners([actorId])).get(actorId) ?? [];
const grants = await kc.listGrants({ ownerId: actorId });
const asks = (await kc.listAsks({ ownerId: actorId })).filter((ask) => ask.status === "pending");
- const usage = deps.credentialUsage
- ? (
- await Promise.all(
- credentials.map(async (credential) => {
- const rows = await deps.credentialUsage!.list({
- slug: `keychain:${credential.service}:${credential.id}`,
- limit: 20,
- });
- return rows.map((row) => ({ ...row, credentialId: credential.id }));
- }),
- )
- )
- .flat()
- .sort((a, b) => b.ts - a.ts)
- .slice(0, 50)
- : [];
const scopeNames = await resolveScopeNames(app, deps, [
...grants.map((grant) => grant.audienceScopeId),
...asks.map((ask) => ask.requesterScopeId),
- ...usage.map((row) => row.scopeLabel),
]);
- return sendJson(res, 200, { credentials, connectorCredentials, grants, asks, usage, scopeNames });
+ return sendJson(res, 200, { credentials, connectorCredentials, grants, asks, scopeNames });
}
if (method === "DELETE" && pathname.startsWith("/v1/keychain/credentials/")) {
diff --git a/src/persistence/pg-pool.ts b/src/persistence/pg-pool.ts
index e085e5c7f..467feaaa7 100644
--- a/src/persistence/pg-pool.ts
+++ b/src/persistence/pg-pool.ts
@@ -173,7 +173,9 @@ export function assertOneStatement(stmt: string): void {
}
export function concurrentIndexName(stmt: string): string | undefined {
- return /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+IF\s+NOT\s+EXISTS\s+([a-z_][a-z0-9_$]*)\b/i.exec(stmt)?.[1];
+ return /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+CONCURRENTLY\s+IF\s+NOT\s+EXISTS\s+([a-z_][a-z0-9_$]*)\s+ON\s/i.exec(
+ stmt,
+ )?.[1];
}
export function pgMigrationChecksum(statements: readonly string[]): string {
@@ -227,6 +229,42 @@ export async function applyPgMigrations(pool: Pool, migrations: readonly PgMigra
}
continue;
}
+ if (migration.statements.some((statement) => concurrentIndexName(statement))) {
+ if (migration.legacyId || !migration.statements.every((statement) => concurrentIndexName(statement))) {
+ throw new Error(`pg-pool: concurrent index migration ${migration.id} must contain only concurrent indexes`);
+ }
+ for (const statement of migration.statements) {
+ const name = concurrentIndexName(statement)!;
+ const table =
+ /\sON\s+(?:ONLY\s+)?([a-z_][a-z0-9_$]*(?:\.[a-z_][a-z0-9_$]*)?)\s*(?:USING\s+[a-z_]+\s*)?\(/i.exec(
+ statement,
+ )?.[1];
+ if (!table) throw new Error(`pg-pool: unsupported concurrent index target in ${migration.id}`);
+ const inspect = () =>
+ client.query<{ indisvalid: boolean; indisready: boolean; same_table: boolean; qualified_name: string }>(
+ `SELECT i.indisvalid, i.indisready, i.indrelid = t.oid AS same_table,
+ quote_ident(n.nspname) || '.' || quote_ident(c.relname) AS qualified_name
+ FROM pg_class t JOIN pg_namespace n ON n.oid = t.relnamespace
+ JOIN pg_class c ON c.relnamespace = n.oid AND c.relname = $2
+ JOIN pg_index i ON i.indexrelid = c.oid WHERE t.oid = to_regclass($1)`,
+ [table.toLowerCase(), name.toLowerCase()],
+ );
+ const existing = (await inspect()).rows[0];
+ if (existing && !existing.same_table)
+ throw new Error(`pg-pool: concurrent index ${name} belongs to a different table`);
+ if (existing?.indisvalid === false) await client.query(`DROP INDEX CONCURRENTLY ${existing.qualified_name}`);
+ await client.query(statement);
+ const built = (await inspect()).rows[0];
+ if (!built?.indisvalid || !built.indisready || !built.same_table) {
+ throw new Error(`pg-pool: concurrent index ${name} is not ready and valid`);
+ }
+ }
+ await client.query(`INSERT INTO ${PG_MIGRATIONS_TABLE}(id, checksum) VALUES ($1, $2)`, [
+ migration.id,
+ migration.checksum,
+ ]);
+ continue;
+ }
await client.query("BEGIN");
try {
let adopted = false;
diff --git a/test/keychain.test.ts b/test/keychain.test.ts
index 49b124458..bdbfe4275 100644
--- a/test/keychain.test.ts
+++ b/test/keychain.test.ts
@@ -1324,7 +1324,7 @@ describe("/v1/keychain routes (capability-authed)", () => {
);
});
- it("overview joins owned credential metadata to grants and recent audited use without secret values", async () => {
+ it("overview returns metadata and grants without querying usage history or exposing secrets", async (t) => {
const owner = "OVERVIEW_OWNER";
const { credential } = (await (
await post(
@@ -1341,12 +1341,17 @@ describe("/v1/keychain routes (capability-authed)", () => {
const { grant } = (await granted.json()) as any;
await post("/v1/keychain/use", { grant: grant.id }, await capFor(owner, "channel:C_OVERVIEW"));
+ const usage = t.mock.method(built.credentialUsage, "list", async () => {
+ throw new Error("usage history is unavailable");
+ });
+
const overview = await get("/v1/keychain/overview", await capFor(owner));
assert.equal(overview.status, 200);
const body = (await overview.json()) as any;
assert.equal(body.credentials[0].id, credential.id);
assert.equal(body.grants[0].purpose, "deploy the reporting app");
- assert.equal(body.usage[0].credentialId, credential.id);
+ assert.equal(usage.mock.callCount(), 0);
+ assert.ok(!Object.hasOwn(body, "usage"));
assert.ok(!JSON.stringify(body).includes("never-return-this"));
});
diff --git a/test/postgres-credential-usage-sink.test.ts b/test/postgres-credential-usage-sink.test.ts
index 365f163a8..74ade8a93 100644
--- a/test/postgres-credential-usage-sink.test.ts
+++ b/test/postgres-credential-usage-sink.test.ts
@@ -68,3 +68,27 @@ test("pg credential-usage sink: survives a fresh sink over the same table (durab
const rows = await reopened.list({ limit: 100 });
assert.ok(rows.length >= 2, "calls written by a prior sink instance are still readable");
});
+
+test("pg credential-usage sink: sparse and absent credentials use an ordered index", { skip }, async () => {
+ const pg = (await import("pg")).default;
+ const p = new pg.Pool({ connectionString: URL });
+ try {
+ await p.query(`INSERT INTO credential_usage(ts, slug, host, status, scope_label, principal_id)
+ SELECT n, 'unrelated-' || n, 'example.com', 'ok', 'personal:other', 'other'
+ FROM generate_series(1, 20000) n`);
+ await p.query("ANALYZE credential_usage");
+ for (const slug of ["serp", "missing-credential"]) {
+ const result = await p.query(
+ "EXPLAIN (ANALYZE, FORMAT JSON) SELECT ts, slug, host, status, upstream_status, scope_label, principal_id FROM credential_usage WHERE slug = $1 ORDER BY ts DESC, id DESC LIMIT 20",
+ [slug],
+ );
+ const plan = result.rows[0]["QUERY PLAN"][0].Plan;
+ const scan = plan.Plans[0];
+ assert.equal(scan["Node Type"], "Index Scan");
+ assert.equal(scan["Index Name"], "credential_usage_by_slug_ts_id");
+ assert.equal(scan["Rows Removed by Filter"] ?? 0, 0);
+ }
+ } finally {
+ await p.end();
+ }
+});
diff --git a/test/postgres-migrations.test.ts b/test/postgres-migrations.test.ts
index 243b2ef2b..6d77f05cf 100644
--- a/test/postgres-migrations.test.ts
+++ b/test/postgres-migrations.test.ts
@@ -18,6 +18,105 @@ function names(): { id: string; table: string } {
return { id: `test/integration/${suffix}/0001`, table: `migration_test_${suffix}` };
}
+test("concurrent index migrations adopt prebuilt indexes and serialize retries", { skip }, async () => {
+ const a = new pg.Pool({ connectionString: databaseUrl });
+ const b = new pg.Pool({ connectionString: databaseUrl });
+ const { id, table } = names();
+ const index = `${table}_idx`;
+ const migration = definePgMigration(id, [`CREATE INDEX CONCURRENTLY IF NOT EXISTS ${index} ON ${table}(value)`]);
+ try {
+ await a.query(`CREATE TABLE ${table}(value INT NOT NULL)`);
+ await a.query(`CREATE INDEX CONCURRENTLY ${index} ON ${table}(value)`);
+ const original = await a.query("SELECT to_regclass($1)::oid AS oid", [index]);
+ await Promise.all([applyPgMigrations(a, [migration]), applyPgMigrations(b, [migration])]);
+ const adopted = await a.query("SELECT to_regclass($1)::oid AS oid", [index]);
+ assert.deepEqual(adopted.rows, original.rows);
+ const ledger = await a.query(`SELECT checksum FROM ${PG_MIGRATIONS_TABLE} WHERE id = $1`, [id]);
+ assert.deepEqual(ledger.rows, [{ checksum: migration.checksum }]);
+ } finally {
+ await a.query(`DROP TABLE IF EXISTS ${table}`);
+ await a.query(`DELETE FROM ${PG_MIGRATIONS_TABLE} WHERE id = $1`, [id]);
+ await Promise.all([a.end(), b.end()]);
+ }
+});
+
+test("failed concurrent index builds are unrecorded and rebuild invalid indexes on retry", { skip }, async () => {
+ const pool = new pg.Pool({ connectionString: databaseUrl });
+ const { id, table } = names();
+ const index = `${table}_idx`;
+ const migration = definePgMigration(id, [
+ `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ${index} ON ${table}(value)`,
+ ]);
+ try {
+ await pool.query(`CREATE TABLE ${table}(value INT NOT NULL)`);
+ await pool.query(`INSERT INTO ${table} VALUES (1), (1)`);
+ await assert.rejects(applyPgMigrations(pool, [migration]), /could not create unique index/);
+ const failed = await pool.query("SELECT indisvalid FROM pg_index WHERE indexrelid=to_regclass($1)", [index]);
+ assert.deepEqual(failed.rows, [{ indisvalid: false }]);
+ assert.equal((await pool.query(`SELECT 1 FROM ${PG_MIGRATIONS_TABLE} WHERE id=$1`, [id])).rowCount, 0);
+ await pool.query(`DELETE FROM ${table}`);
+ await pool.query(`INSERT INTO ${table} VALUES (1), (2)`);
+ await applyPgMigrations(pool, [migration]);
+ const repaired = await pool.query("SELECT indisvalid FROM pg_index WHERE indexrelid=to_regclass($1)", [index]);
+ assert.deepEqual(repaired.rows, [{ indisvalid: true }]);
+ assert.equal((await pool.query(`SELECT 1 FROM ${PG_MIGRATIONS_TABLE} WHERE id=$1`, [id])).rowCount, 1);
+ } finally {
+ await pool.query(`DROP TABLE IF EXISTS ${table}`);
+ await pool.query(`DELETE FROM ${PG_MIGRATIONS_TABLE} WHERE id = $1`, [id]);
+ await pool.end();
+ }
+});
+
+test("concurrent index migrations reject mixed transactional statements before execution", { skip }, async () => {
+ const pool = new pg.Pool({ connectionString: databaseUrl });
+ const { id, table } = names();
+ try {
+ await assert.rejects(
+ applyPgMigrations(pool, [
+ definePgMigration(id, [
+ `CREATE TABLE ${table}(value INT)`,
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_idx ON ${table}(value)`,
+ ]),
+ ]),
+ /must contain only concurrent indexes/,
+ );
+ assert.equal((await pool.query("SELECT to_regclass($1) AS name", [table])).rows[0].name, null);
+ } finally {
+ await pool.end();
+ }
+});
+
+test("concurrent index retry resolves indexes in the target table schema", { skip }, async () => {
+ const pool = new pg.Pool({ connectionString: databaseUrl });
+ const { id, table } = names();
+ const schema = `s_${table}`;
+ const index = `${table}_idx`;
+ const migration = definePgMigration(id, [
+ `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS ${index} ON ${schema}.events(value)`,
+ ]);
+ try {
+ await pool.query(`CREATE SCHEMA ${schema}`);
+ await pool.query(`CREATE TABLE ${schema}.events(value INT)`);
+ await pool.query(`CREATE TABLE ${table}(value INT)`);
+ await pool.query(`CREATE INDEX ${index} ON ${table}(value)`);
+ const original = await pool.query("SELECT to_regclass($1)::oid AS oid", [index]);
+ await pool.query(`INSERT INTO ${schema}.events VALUES (1), (1)`);
+ await assert.rejects(applyPgMigrations(pool, [migration]), /could not create unique index/);
+ await pool.query(`DELETE FROM ${schema}.events`);
+ await applyPgMigrations(pool, [migration]);
+ assert.deepEqual((await pool.query("SELECT to_regclass($1)::oid AS oid", [index])).rows, original.rows);
+ const repaired = await pool.query("SELECT indisvalid FROM pg_index WHERE indexrelid=to_regclass($1)", [
+ `${schema}.${index}`,
+ ]);
+ assert.deepEqual(repaired.rows, [{ indisvalid: true }]);
+ } finally {
+ await pool.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);
+ await pool.query(`DROP TABLE IF EXISTS ${table}`);
+ await pool.query(`DELETE FROM ${PG_MIGRATIONS_TABLE} WHERE id=$1`, [id]);
+ await pool.end();
+ }
+});
+
test("concurrent migrators apply one version exactly once", { skip }, async () => {
const a = new pg.Pool({ connectionString: databaseUrl });
const b = new pg.Pool({ connectionString: databaseUrl });
diff --git a/test/scoped-event-sink-schema.test.ts b/test/scoped-event-sink-schema.test.ts
index 3d15487b7..b0fb3eb2e 100644
--- a/test/scoped-event-sink-schema.test.ts
+++ b/test/scoped-event-sink-schema.test.ts
@@ -93,6 +93,7 @@ test("every released scoped-event sink pins its 0001 checksum", () => {
registeredPgMigrations(url).map((m) => m.id),
[
"admin/scoped-events/credential_usage/0001",
+ "admin/scoped-events/credential_usage/0002",
"admin/scoped-events/egress_events/0001",
"admin/scoped-events/error_events/0001",
],