Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

15.9 KB
Raw Blame
import { createPgPool, type PgPool, type PoolClient } from "./pg-pool.ts";
import { pgTextSafe } from "../util/text.ts";

export interface DurableMapSelect<T, K extends Extract<keyof T, string>> {
  omit?: readonly K[];
  pickNested?: Partial<Record<Extract<keyof T, string>, readonly string[]>>;
  where?: { field: Extract<keyof T, string>; anyOfFold: readonly string[] };
  limit?: number;
  afterId?: string;
}

export interface DurableMap<T> {
  all(): Promise<T[]>;
  entries(): Promise<Array<[string, T]>>;
  select<K extends Extract<keyof T, string> = never>(query: DurableMapSelect<T, K>): Promise<Array<Omit<T, K>>>;
  selectEntries<K extends Extract<keyof T, string> = never>(
    query: DurableMapSelect<T, K>,
  ): Promise<Array<[string, Omit<T, K>]>>;
  get(id: string): Promise<T | null>;
  put(id: string, value: T): Promise<void>;
  putIfAbsent(id: string, value: T): Promise<T>;
  insertIfAbsent?(id: string, value: T): Promise<boolean>;
  merge(id: string, patch: Partial<T>): Promise<T | null>;
  update?(id: string, fn: (value: T) => T): Promise<T | null>;
  mutateMany(updates: readonly { id: string; apply: (value: T | null) => T }[]): Promise<T[]>;
  deleteIf?(id: string, predicate: (value: T) => boolean): Promise<boolean>;
  delete(id: string): Promise<void>;
  take(id: string): Promise<T | null>;
}

/**
 * Serialize for a Postgres jsonb column. jsonb rejects two things a JS string
 * happily carries: NUL (\u0000) and unpaired surrogate halves — and qm's own
 * truncation helpers can manufacture the latter by slicing mid-emoji. The
 * memory map accepts those values, so production diverged from every
 * in-memory test. Sanitize at the serialization boundary: drop NULs and
 * replace lone surrogates with U+FFFD, recursively, only when a string
 * actually needs it.
 */
function jsonbSafe(value: unknown): unknown {
  if (typeof value === "string") return pgTextSafe(value);
  if (Array.isArray(value)) return value.map(jsonbSafe);
  if (value && typeof value === "object") {
    const out: Record<string, unknown> = Object.create(null);
    for (const [k, v] of Object.entries(value)) out[pgTextSafe(k)] = jsonbSafe(v);
    return out;
  }
  return value;
}

export function jsonbStringify(value: unknown): string {
  return JSON.stringify(jsonbSafe(value));
}

function fieldText(value: unknown): string | null {
  if (value == null) return null;
  return typeof value === "string" ? value : JSON.stringify(value);
}

function isAscii(text: string): boolean {
  for (const ch of text) if (ch.codePointAt(0)! > 0x7f) return false;
  return true;
}

export function selectEntries<T, K extends Extract<keyof T, string>>(
  entries: ReadonlyArray<readonly [string, T]>,
  query: DurableMapSelect<T, K>,
): Array<[string, Omit<T, K>]> {
  const folded = query.where ? new Set(query.where.anyOfFold.map((v) => v.toLowerCase())) : null;
  const out: Array<[string, Omit<T, K>]> = [];
  for (const [id, value] of entries) {
    if (query.limit !== undefined && out.length >= query.limit) break;
    if (query.afterId !== undefined && id <= query.afterId) continue;
    if (folded && query.where) {
      const text = fieldText((value as Record<string, unknown>)[query.where.field]);
      if (text === null || !(folded.has(text.toLowerCase()) || !isAscii(text))) continue;
    }
    const projected = structuredClone(value) as Record<string, unknown>;
    for (const [field, keys] of Object.entries(query.pickNested ?? {}) as Array<[string, readonly string[]]>) {
      const nested = projected[field];
      if (nested && typeof nested === "object" && !Array.isArray(nested))
        projected[field] = Object.fromEntries(Object.entries(nested).filter(([key]) => keys.includes(key)));
    }
    for (const key of query.omit ?? []) delete projected[key];
    out.push([id, projected as Omit<T, K>]);
  }
  return out;
}

function applyPatch<T>(value: T, patch: Partial<T>): T {
  const next = { ...value } as Record<string, unknown>;
  for (const [k, v] of Object.entries(patch)) {
    if (v === undefined) delete next[k];
    else next[k] = v;
  }
  return next as T;
}

export function createMemoryMap<T>(): DurableMap<T> {
  const m = new Map<string, T>();
  const sortedEntries = () =>
    [...m.entries()].sort(([a], [b]) => {
      if (a < b) return -1;
      if (a > b) return 1;
      return 0;
    });
  return {
    async all() {
      return sortedEntries().map(([, v]) => v);
    },
    async entries() {
      return sortedEntries();
    },
    async select(query) {
      return selectEntries(sortedEntries(), query).map(([, value]) => value);
    },
    async selectEntries(query) {
      return selectEntries(sortedEntries(), query);
    },
    async get(id) {
      return m.get(id) ?? null;
    },
    async put(id, value) {
      m.set(id, value);
    },
    async putIfAbsent(id, value) {
      const existing = m.get(id);
      if (existing !== undefined) return existing;
      m.set(id, value);
      return value;
    },
    async insertIfAbsent(id, value) {
      if (m.has(id)) return false;
      m.set(id, value);
      return true;
    },
    async merge(id, patch) {
      const v = m.get(id);
      if (v == null) return null;
      const merged = applyPatch(v, patch);
      m.set(id, merged);
      return merged;
    },
    async update(id, fn) {
      const v = m.get(id);
      if (v == null) return null;
      const next = fn(v);
      m.set(id, next);
      return next;
    },
    async mutateMany(updates) {
      const pending = new Map<string, T>();
      const results = updates.map(({ id, apply }) => {
        const next = apply(structuredClone(pending.get(id) ?? m.get(id) ?? null));
        pending.set(id, next);
        return next;
      });
      for (const [id, value] of pending) m.set(id, value);
      return results;
    },
    async deleteIf(id, predicate) {
      const value = m.get(id);
      if (value === undefined || !predicate(value)) return false;
      m.delete(id);
      return true;
    },
    async delete(id) {
      m.delete(id);
    },
    async take(id) {
      const v = m.get(id) ?? null;
      m.delete(id);
      return v;
    },
  };
}

const VERSIONS_TABLE = "durable_map_versions";

export function createPostgresMap<T>(
  pg: PgPool,
  table: string,
  indexedFields: readonly Extract<keyof T, string>[] = [],
): DurableMap<T> {
  if (!/^[a-z_][a-z0-9_]*$/i.test(table)) throw new Error(`invalid table name: ${table}`);
  const migration = {
    id: `durable-map/${table}/0001`,
    statements: [
      `CREATE TABLE IF NOT EXISTS ${table} (id TEXT PRIMARY KEY, json JSONB NOT NULL)`,
      `CREATE TABLE IF NOT EXISTS ${VERSIONS_TABLE} (tbl TEXT PRIMARY KEY, v BIGINT NOT NULL)`,
    ],
  };
  const migrations = [
    migration,
    ...indexedFields.flatMap((field) => {
      if (!/^[a-z_][a-z0-9_]*$/i.test(field)) throw new Error(`invalid index field: ${field}`);
      return [
        {
          id: `durable-map/${table}/lookup-${field.toLowerCase()}-drop`,
          statements: [`SET LOCAL lock_timeout = '3s'`, `DROP INDEX IF EXISTS ${table}_${field.toLowerCase()}_fold`],
        },
        {
          id: `durable-map/${table}/lookup-${field.toLowerCase()}-hash`,
          statements: [
            `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_fold
           ON ${table} USING hash (lower(json->>'${field}'))`,
          ],
        },
        {
          id: `durable-map/${table}/select-${field.toLowerCase()}`,
          statements: [
            `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_fold
           ON ${table} (lower(json->>'${field}'))`,
            `CREATE INDEX CONCURRENTLY IF NOT EXISTS ${table}_${field.toLowerCase()}_unicode
           ON ${table} (id) WHERE json->>'${field}' ~ '[^\\x01-\\x7f]'`,
          ],
        },
        {
          id: `durable-map/${table}/stats-${field.toLowerCase()}`,
          statements: [
            `SET LOCAL lock_timeout = '3s'`,
            `CREATE STATISTICS IF NOT EXISTS ${table}_${field.toLowerCase()}_stats ON (json->>'${field}') FROM ${table}`,
            `ANALYZE ${table}`,
          ],
        },
      ];
    }),
  ];
  for (const definition of migrations) pg.registerMigration(definition);
  let readyP: Promise<void> | null = null;
  function ready(): Promise<void> {
    if (!readyP) {
      readyP = (async () => {
        for (const definition of migrations) await pg.migrate(definition);
      })().catch((e) => {
        readyP = null;
        throw e;
      });
    }
    return readyP;
  }
  const bumpSql = `INSERT INTO ${VERSIONS_TABLE} (tbl, v) VALUES ('${table}', 1)
     ON CONFLICT (tbl) DO UPDATE SET v = ${VERSIONS_TABLE}.v + 1`;
  async function withBump<R>(fn: (client: PoolClient) => Promise<R>): Promise<R> {
    await ready();
    const client = await (await pg.pool()).connect();
    try {
      await client.query("BEGIN");
      await client.query(bumpSql);
      const result = await fn(client);
      await client.query("COMMIT");
      return result;
    } catch (e) {
      await client.query("ROLLBACK");
      throw e;
    } finally {
      client.release();
    }
  }
  const CACHE_MAX_AGE_MS = 15_000;
  let cache: { v: string; fetchedAt: number; entries: Array<[string, T]> } | null = null;
  async function snapshot(): Promise<Array<[string, T]>> {
    await ready();
    const vRows = await pg.q(`SELECT v FROM ${VERSIONS_TABLE} WHERE tbl = '${table}'`);
    const v = vRows.length ? String(vRows[0]!.v) : "";
    if (!cache || cache.v !== v || Date.now() - cache.fetchedAt > CACHE_MAX_AGE_MS) {
      const rows = await pg.q(`SELECT id, json FROM ${table} ORDER BY id`);
      cache = { v, fetchedAt: Date.now(), entries: rows.map((row) => [row.id as string, row.json as T]) };
    }
    return structuredClone(cache.entries);
  }
  async function selectRows<K extends Extract<keyof T, string>>(
    query: DurableMapSelect<T, K>,
  ): Promise<Array<[string, Omit<T, K>]>> {
    await ready();
    const params: unknown[] = [[...(query.omit ?? [])]];
    let projection = "json";
    for (const [field, keys] of Object.entries(query.pickNested ?? {})) {
      params.push(field, keys);
      const fieldParam = `$${params.length - 1}`;
      const keysParam = `$${params.length}`;
      const nested = `CASE WHEN jsonb_typeof(json->${fieldParam}) = 'object' THEN
          COALESCE((SELECT jsonb_object_agg(key, value) FROM jsonb_each(json->${fieldParam})
            WHERE key = ANY(${keysParam}::text[])), '{}'::jsonb) ELSE json->${fieldParam} END`;
      projection = `(${projection} || CASE WHEN json ? ${fieldParam} THEN
          jsonb_build_object(${fieldParam}::text, ${nested}) ELSE '{}'::jsonb END)`;
    }
    let sql = `SELECT id, ${projection} - $1::text[] AS json FROM ${table}`;
    if (query.where) {
      const field = query.where.field;
      if (!/^[a-z_][a-z0-9_]*$/i.test(field)) throw new Error(`invalid select field: ${field}`);
      params.push(query.where.anyOfFold.map((v) => v.toLowerCase()));
      sql += ` WHERE id IN (
          SELECT id FROM ${table} WHERE lower(json->>'${field}') = ANY($${params.length}::text[])
          UNION
          SELECT id FROM ${table} WHERE json->>'${field}' ~ '[^\\x01-\\x7f]')`;
    }
    if (query.afterId !== undefined) {
      params.push(query.afterId);
      sql += `${query.where ? " AND" : " WHERE"} id > $${params.length}`;
    }
    sql += " ORDER BY id";
    if (query.limit !== undefined) {
      params.push(query.limit);
      sql += ` LIMIT $${params.length}`;
    }
    const rows = await pg.q(sql, params);
    return rows.map((row) => [row.id as string, row.json as Omit<T, K>]);
  }
  return {
    async all() {
      return (await snapshot()).map(([, value]) => value);
    },
    async entries() {
      return snapshot();
    },
    async select<K extends Extract<keyof T, string> = never>(query: DurableMapSelect<T, K>) {
      return (await selectRows(query)).map(([, value]) => value);
    },
    selectEntries: selectRows,
    async get(id) {
      await ready();
      const rows = await pg.q(`SELECT json FROM ${table} WHERE id = $1`, [id]);
      return rows.length ? (rows[0]!.json as T) : null;
    },
    async put(id, value) {
      await withBump((client) =>
        client.query(
          `INSERT INTO ${table} (id, json) VALUES ($1, $2)
           ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json`,
          [id, jsonbStringify(value)],
        ),
      );
    },
    async putIfAbsent(id, value) {
      const res = await withBump((client) =>
        client.query(
          `INSERT INTO ${table} (id, json) VALUES ($1, $2)
           ON CONFLICT (id) DO UPDATE SET json = ${table}.json
           RETURNING json`,
          [id, jsonbStringify(value)],
        ),
      );
      return res.rows[0]!.json as T;
    },
    async insertIfAbsent(id, value) {
      const inserted = await withBump((client) =>
        client.query(
          `INSERT INTO ${table} (id, json) VALUES ($1, $2)
           ON CONFLICT (id) DO NOTHING`,
          [id, jsonbStringify(value)],
        ),
      );
      return (inserted.rowCount ?? 0) > 0;
    },
    async merge(id, patch) {
      const entries = Object.entries(patch);
      const removeKeys = entries.filter(([, v]) => v === undefined).map(([k]) => k);
      const set = Object.fromEntries(entries.filter(([, v]) => v !== undefined));
      const res = await withBump((client) =>
        client.query(`UPDATE ${table} SET json = (json - $2::text[]) || $3::jsonb WHERE id = $1 RETURNING json`, [
          id,
          removeKeys,
          jsonbStringify(set),
        ]),
      );
      return res.rows.length ? (res.rows[0]!.json as T) : null;
    },
    async update(id, fn) {
      return withBump(async (client) => {
        const current = await client.query(`SELECT json FROM ${table} WHERE id = $1 FOR UPDATE`, [id]);
        if (!current.rows[0]) return null;
        const next = fn(current.rows[0].json as T);
        await client.query(`UPDATE ${table} SET json = $2 WHERE id = $1`, [id, jsonbStringify(next)]);
        return next;
      });
    },
    async mutateMany(updates) {
      if (!updates.length) return [];
      return withBump(async (client) => {
        const rows = await client.query(
          `SELECT id, json FROM ${table} WHERE id = ANY($1::text[]) ORDER BY id FOR UPDATE`,
          [updates.map(({ id }) => id)],
        );
        const current = new Map(rows.rows.map((row) => [row.id as string, row.json as T]));
        const pending = new Map<string, T>();
        const results = updates.map(({ id, apply }) => {
          const next = apply(structuredClone(pending.get(id) ?? current.get(id) ?? null));
          pending.set(id, next);
          return next;
        });
        await client.query(
          `INSERT INTO ${table} (id, json) SELECT id, json FROM jsonb_to_recordset($1::jsonb) AS batch(id text, json jsonb)
           ON CONFLICT (id) DO UPDATE SET json = EXCLUDED.json`,
          [jsonbStringify([...pending].map(([id, json]) => ({ id, json })))],
        );
        return results;
      });
    },
    async deleteIf(id, predicate) {
      return withBump(async (client) => {
        const current = await client.query(`SELECT json FROM ${table} WHERE id = $1 FOR UPDATE`, [id]);
        if (!current.rows[0] || !predicate(current.rows[0].json as T)) return false;
        await client.query(`DELETE FROM ${table} WHERE id = $1`, [id]);
        return true;
      });
    },
    async delete(id) {
      await withBump((client) => client.query(`DELETE FROM ${table} WHERE id = $1`, [id]));
    },
    async take(id) {
      const res = await withBump((client) => client.query(`DELETE FROM ${table} WHERE id = $1 RETURNING json`, [id]));
      return res.rows.length ? (res.rows[0]!.json as T) : null;
    },
  };
}

export interface PostgresArtifactMaps {
  map<T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T>;
  pool: PgPool;
}

export function createPostgresMapFactory(connectionString: string): PostgresArtifactMaps {
  const pg = createPgPool(connectionString);
  return {
    map: <T>(table: string, indexedFields?: readonly Extract<keyof T, string>[]): DurableMap<T> =>
      createPostgresMap<T>(pg, table, indexedFields),
    pool: pg,
  };
}