Skip to content
Python
15.9 KB
Raw Blame
import { spawn } from "node:child_process";
import * as fs from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, expect, it, vi } from "vitest";
import {
  type BatchAdapters,
  type BatchJournal,
  ConnectionBatch,
  type ConnectionPlan,
  connectionPlanDigest,
} from "./connection-batch.js";
import { PersistentBatchLedger } from "./connection-batch-ledger.js";

vi.mock("node:fs/promises", async (original) => {
  const actual = await original<typeof import("node:fs/promises")>();
  return { ...actual, rename: vi.fn(actual.rename), open: vi.fn(actual.open) };
});

const owner = { userId: "local", runtimeId: "verified-data-instance" };
const roots: string[] = [];
const digest = "a".repeat(64);
const source = {
  sourceId: "cursor",
  rootId: "root",
  rootPath: "/fixture",
  grantRevision: 1,
};
const host = {
  hostId: "cursor",
  layoutDigest: digest,
  installationId: null,
  permissionRevision: 0,
  installationDigest: null,
};
function journal(revision = 1): BatchJournal {
  return {
    schemaVersion: 1,
    ...owner,
    operationId: "operation-one",
    planDigest: digest,
    revision,
    items: [
      {
        id: "install",
        kind: "install",
        intentDigest: digest,
        state: "pending",
        receipt: null,
      },
    ],
  };
}
async function fixture() {
  const root = await fs.realpath(
    await fs.mkdtemp(path.join(tmpdir(), "batch-ledger-")),
  );
  roots.push(root);
  await fs.chmod(root, 0o700);
  const ledger = await PersistentBatchLedger.initialize(root, owner);
  return { root, ledger, directory: path.join(root, "connection-batches") };
}
async function create(ledger: PersistentBatchLedger) {
  await ledger.exclusive(owner, () =>
    ledger.commit(owner, "operation-one", null, journal()),
  );
}
async function journalPath(directory: string) {
  const names = await fs.readdir(directory);
  return path.join(
    directory,
    names.find((n) => /^[a-f0-9]{64}\.json$/.test(n)) ?? "missing",
  );
}
afterEach(async () => {
  vi.restoreAllMocks();
  await Promise.all(
    roots
      .splice(0)
      .map((root) => fs.rm(root, { recursive: true, force: true })),
  );
});
it("persists private journals, reloads stable owner, and enforces create/update CAS", async () => {
  const { root, ledger, directory } = await fixture();
  expect(await ledger.load(owner, "operation-one")).toBeNull();
  await create(ledger);
  expect((await fs.stat(directory)).mode & 0o777).toBe(0o700);
  expect((await fs.stat(await journalPath(directory))).mode & 0o777).toBe(
    0o600,
  );
  const reopened = await PersistentBatchLedger.open(root, owner);
  expect(await reopened.load(owner, "operation-one")).toEqual(journal());
  await expect(
    reopened.exclusive(owner, () =>
      reopened.commit(owner, "operation-one", null, journal()),
    ),
  ).rejects.toThrow("revision_conflict");
  await reopened.exclusive(owner, () =>
    reopened.commit(owner, "operation-one", 1, journal(2)),
  );
  await expect(
    ledger.exclusive(owner, () =>
      ledger.commit(owner, "operation-one", 1, journal(2)),
    ),
  ).rejects.toThrow("revision_conflict");
  expect((await ledger.load(owner, "operation-one"))?.revision).toBe(2);
});
it("excludes separate executors across all operations, and releases after failure", async () => {
  const { root, ledger } = await fixture();
  const other = await PersistentBatchLedger.open(root, owner);
  let release!: () => void;
  let entered!: () => void;
  const ready = new Promise<void>((r) => {
    entered = r;
  });
  const effect = vi.fn();
  const held = ledger.exclusive(owner, async () => {
    entered();
    await new Promise<void>((r) => {
      release = r;
    });
  });
  await ready;
  await expect(
    other.exclusive(owner, async () => {
      effect();
    }),
  ).rejects.toThrow("recovery_required");
  expect(effect).not.toHaveBeenCalled();
  release();
  await held;
  await expect(
    ledger.exclusive(owner, async () => {
      throw Error("effect failure");
    }),
  ).rejects.toThrow("effect failure");
  await other.exclusive(owner, async () => {
    effect();
  });
  expect(effect).toHaveBeenCalledTimes(1);
});
it("requires explicit initialization and exact owner; owner loss is not a fresh store", async () => {
  const { root, directory, ledger } = await fixture();
  await expect(
    PersistentBatchLedger.open(root, { ...owner, runtimeId: "different" }),
  ).rejects.toThrow("owner_mismatch");
  await expect(
    ledger.load({ ...owner, userId: "other" }, "operation-one"),
  ).rejects.toThrow("owner_mismatch");
  await fs.unlink(path.join(directory, "owner.json"));
  await expect(PersistentBatchLedger.open(root, owner)).rejects.toThrow();
  await expect(PersistentBatchLedger.initialize(root, owner)).rejects.toThrow();
});
it("rejects caller paths, extra fields, bad digest/revision, and preserves successful history", async () => {
  const { ledger } = await fixture();
  await expect(ledger.load(owner, "../../outside")).rejects.toThrow();
  await ledger.exclusive(owner, async () => {
    await expect(
      ledger.commit(owner, "operation-one", null, {
        ...journal(),
        secret: "x",
      } as BatchJournal),
    ).rejects.toThrow();
    await ledger.commit(owner, "operation-one", null, journal());
    await expect(
      ledger.commit(owner, "operation-one", 1, {
        ...journal(2),
        planDigest: "b".repeat(64),
      }),
    ).rejects.toThrow("operation_conflict");
    const succeeded = journal(2);
    succeeded.items[0] = {
      ...succeeded.items[0],
      state: "succeeded",
      receipt: {
        ...owner,
        operationId: "operation-one",
        itemId: "install",
        intentDigest: digest,
        kind: "install",
        host: {
          ...host,
          installationId: "installation-one",
          permissionRevision: 1,
          installationDigest: "b".repeat(64),
        },
      },
    };
    await ledger.commit(owner, "operation-one", 1, succeeded);
    await expect(
      ledger.commit(owner, "operation-one", 2, journal(3)),
    ).rejects.toThrow("history_conflict");
  });
});
it.each(["corrupt", "wide", "symlink", "hardlink", "oversized"])(
  "rejects %s journal instead of replacing it",
  async (kind) => {
    const { ledger, directory } = await fixture();
    await create(ledger);
    const file = await journalPath(directory);
    if (kind === "corrupt") await fs.writeFile(file, "{");
    if (kind === "wide") await fs.chmod(file, 0o644);
    if (kind === "oversized") await fs.writeFile(file, "x".repeat(600000));
    if (kind === "hardlink")
      await fs.link(file, path.join(directory, "linked"));
    if (kind === "symlink") {
      await fs.rename(file, `${file}.old`);
      await fs.symlink(`${file}.old`, file);
    }
    await expect(ledger.load(owner, "operation-one")).rejects.toThrow();
    await expect(
      ledger.exclusive(owner, () =>
        ledger.commit(owner, "operation-one", 1, journal(2)),
      ),
    ).rejects.toThrow();
  },
);
it("refuses directory inode replacement and never steals stale/malformed lock", async () => {
  const { root, ledger, directory } = await fixture();
  await fs.mkdir(path.join(directory, "execution.lock"), { mode: 0o700 });
  await expect(ledger.exclusive(owner, async () => {})).rejects.toThrow(
    "recovery_required",
  );
  await fs.rmdir(path.join(directory, "execution.lock"));
  await fs.rename(directory, `${directory}.old`);
  await fs.mkdir(directory, { mode: 0o700 });
  await expect(ledger.load(owner, "operation-one")).rejects.toThrow(
    "directory_changed",
  );
  expect(await fs.readdir(directory)).toEqual([]);
  await expect(PersistentBatchLedger.open(root, owner)).rejects.toThrow();
});
it("failure before publish preserves old bytes and allows safe reload", async () => {
  const { root, ledger, directory } = await fixture();
  await create(ledger);
  const file = await journalPath(directory),
    before = await fs.readFile(file);
  vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("injected IO"));
  await expect(
    ledger.exclusive(owner, () =>
      ledger.commit(owner, "operation-one", 1, journal(2)),
    ),
  ).rejects.toThrow();
  expect(await fs.readFile(file)).toEqual(before);
  expect(
    await (await PersistentBatchLedger.open(root, owner)).load(
      owner,
      "operation-one",
    ),
  ).toEqual(journal());
});
it("does not permit unlocked commits or mutate journals supplied by callers during awaits", async () => {
  const { ledger } = await fixture();
  await expect(
    ledger.commit(owner, "operation-one", null, journal()),
  ).rejects.toThrow("lock_required");
  await ledger.exclusive(owner, async () => {
    const value = journal();
    const commit = ledger.commit(owner, "operation-one", null, value);
    value.planDigest = "b".repeat(64);
    await commit;
  });
  expect((await ledger.load(owner, "operation-one"))?.planDigest).toBe(digest);
});

it.each([
  "root-mode",
  "store-mode",
  "owner-mode",
  "owner-link",
  "owner-corrupt",
  "store-link",
])("fails closed for %s without fixing user files", async (kind) => {
  const { root, ledger, directory } = await fixture();
  const file = path.join(directory, "owner.json");
  if (kind === "root-mode") await fs.chmod(root, 0o755);
  if (kind === "store-mode") await fs.chmod(directory, 0o755);
  if (kind === "owner-mode") await fs.chmod(file, 0o644);
  if (kind === "owner-corrupt") await fs.writeFile(file, "{}");
  if (kind === "owner-link") await fs.link(file, path.join(directory, "alias"));
  if (kind === "store-link") {
    await fs.rename(directory, `${directory}.old`);
    await fs.symlink(`${directory}.old`, directory);
  }
  await expect(ledger.load(owner, "operation-one")).rejects.toThrow();
  await expect(PersistentBatchLedger.open(root, owner)).rejects.toThrow();
});

it("recovers a real install executor without repeating its external effect", async () => {
  const { root, ledger } = await fixture();
  const plan: ConnectionPlan = {
    schemaVersion: 1,
    ...owner,
    operationId: "effect",
    items: [
      {
        id: "install",
        kind: "install",
        sources: [source],
        host,
        previewFingerprint: digest,
      },
    ],
  };
  const apply = vi.fn(async (_item, context) => ({
    status: "succeeded" as const,
    receipt: {
      userId: context.userId,
      runtimeId: context.runtimeId,
      operationId: context.operationId,
      itemId: context.itemId,
      intentDigest: context.intentDigest,
      kind: "install" as const,
      host: {
        ...host,
        installationId: "installation-one",
        permissionRevision: 1,
        installationDigest: "b".repeat(64),
      },
    },
  }));
  const adapters: BatchAdapters = {
    inspect: async () => ({
      status: "ready",
      sources: [source],
      host,
      previewFingerprint: digest,
    }),
    install: {
      apply,
      reconcile: async () => ({ status: "unknown" }),
    },
    readGrant: {
      apply: async () => ({ status: "unknown" }),
      reconcile: async () => ({ status: "unknown" }),
    },
  };
  const options = {
    mode: "execute" as const,
    userConfirmed: true,
    planDigest: connectionPlanDigest(plan),
  };
  expect(
    (await new ConnectionBatch(owner, adapters, ledger).run(plan, options))
      .state,
  ).toBe("completed");
  const reopened = await PersistentBatchLedger.open(root, owner);
  expect(
    (await new ConnectionBatch(owner, adapters, reopened).run(plan, options))
      .state,
  ).toBe("completed");
  expect(apply).toHaveBeenCalledTimes(1);
});

it("serializes concurrent commits inside the same admitted lease", async () => {
  const { ledger } = await fixture();
  await create(ledger);
  await ledger.exclusive(owner, async () => {
    const results = await Promise.allSettled([
      ledger.commit(owner, "operation-one", 1, journal(2)),
      ledger.commit(owner, "operation-one", 1, journal(2)),
    ]);
    expect(results.map((r) => r.status)).toEqual(["fulfilled", "rejected"]);
  });
  expect((await ledger.load(owner, "operation-one"))?.revision).toBe(2);
});

it("a separate Node process cannot execute while the data-instance lease is held", async () => {
  const { root, ledger } = await fixture();
  const module = fileURLToPath(
    new URL("./connection-batch-ledger.ts", import.meta.url),
  );
  await ledger.exclusive(owner, async () => {
    const child = spawn(
      process.execPath,
      [
        "--experimental-transform-types",
        "--input-type=module",
        "-e",
        `
      import {PersistentBatchLedger} from ${JSON.stringify(module)};
      const owner=${JSON.stringify(owner)};
      const ledger=await PersistentBatchLedger.open(${JSON.stringify(root)},owner);
      try { await ledger.exclusive(owner,async()=>{console.log('BAD_EFFECT')}); }
      catch(error){console.log(error.message);}
    `,
      ],
      { stdio: ["ignore", "pipe", "pipe"] },
    );
    let stdout = "",
      stderr = "";
    child.stdout.on("data", (chunk) => {
      stdout += chunk;
    });
    child.stderr.on("data", (chunk) => {
      stderr += chunk;
    });
    const code = await new Promise<number | null>((resolve, reject) => {
      child.once("exit", resolve);
      child.once("error", reject);
    });
    expect(code, stderr).toBe(0);
    expect(stdout.trim()).toBe("connection_batch_ledger_recovery_required");
  });
});

it("rejects receipt payload extras and unknown grant scopes without replacing a journal", async () => {
  const { ledger } = await fixture();
  await create(ledger);
  const next = journal(2);
  next.items[0] = {
    ...next.items[0],
    state: "succeeded",
    receipt: {
      ...owner,
      operationId: "operation-one",
      itemId: "install",
      intentDigest: digest,
      kind: "install",
      host: {
        ...host,
        installationId: "installation-one",
        permissionRevision: 1,
        installationDigest: "b".repeat(64),
      },
      credential: "MUST_NOT_STORE",
    } as never,
  };
  await expect(
    ledger.exclusive(owner, () =>
      ledger.commit(owner, "operation-one", 1, next),
    ),
  ).rejects.toThrow();
  expect(await ledger.load(owner, "operation-one")).toEqual(journal());
});

it("a known dead PID lock is still recovery-required, never age-based stolen", async () => {
  const { ledger, directory } = await fixture();
  const lock = path.join(directory, "execution.lock");
  await fs.mkdir(lock, { mode: 0o700 });
  await fs.writeFile(
    path.join(lock, "owner.json"),
    JSON.stringify({ ...owner, pid: 2147483647, nonce: "old" }),
    { mode: 0o600 },
  );
  const effect = vi.fn();
  await expect(
    ledger.exclusive(owner, async () => {
      effect();
    }),
  ).rejects.toThrow("recovery_required");
  expect(effect).not.toHaveBeenCalled();
  expect(
    JSON.parse(await fs.readFile(path.join(lock, "owner.json"), "utf8")).nonce,
  ).toBe("old");
});

it("post-publication directory fsync failure preserves published journal and fences further execution", async () => {
  const { root, ledger, directory } = await fixture();
  await create(ledger);
  const file = await journalPath(directory);
  const actual =
    await vi.importActual<typeof import("node:fs/promises")>(
      "node:fs/promises",
    );
  vi.mocked(fs.open).mockImplementation(async (...args) => {
    const handle = await actual.open(...args);
    if (args[0] === directory) {
      const sync = handle.sync.bind(handle);
      handle.sync = async () => {
        if (JSON.parse(await actual.readFile(file, "utf8")).revision === 2)
          throw Error("injected fsync failure");
        await sync();
      };
    }
    return handle;
  });
  await expect(
    ledger.exclusive(owner, () =>
      ledger.commit(owner, "operation-one", 1, journal(2)),
    ),
  ).rejects.toThrow("recovery_required");
  expect(JSON.parse(await actual.readFile(file, "utf8")).revision).toBe(2);
  vi.mocked(fs.open).mockImplementation(actual.open);
  const reopened = await PersistentBatchLedger.open(root, owner);
  expect((await reopened.load(owner, "operation-one"))?.revision).toBe(2);
  await expect(reopened.exclusive(owner, async () => {})).rejects.toThrow(
    "recovery_required",
  );
});