Skip to content
Python
16.4 KB
Raw Blame
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import * as fs from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { afterEach, expect, it, vi } from "vitest";
import type { BackgroundOperationSnapshot } from "./background-operation.js";
import { PersistentBackgroundOperationLedger } from "./background-operation-ledger.js";

vi.mock("node:fs/promises", async (original) => {
  const actual = await original<typeof import("node:fs/promises")>();
  return {
    ...actual,
    open: vi.fn(actual.open),
    link: vi.fn(actual.link),
    rename: vi.fn(actual.rename),
  };
});
const owner = { userId: "local", runtimeId: "stable-data-instance" };
const roots: string[] = [];
function snapshot(
  revision = 1,
  state: BackgroundOperationSnapshot["operations"][number]["state"] = "prepared",
): BackgroundOperationSnapshot {
  return {
    schemaVersion: 1,
    ...owner,
    revision,
    operations: [
      {
        operationId: "19e60957-ccfe-4d79-8d6b-c109c0b5f3fa",
        expectedRevision: 0,
        action: "enable",
        planDigest: "a".repeat(64),
        artifactDigest: "b".repeat(64),
        state,
        code: state === "unknown" ? "interrupted" : null,
      },
    ],
  };
}
async function root() {
  const value = await fs.realpath(
    await fs.mkdtemp(path.join(tmpdir(), "background-ledger-test-")),
  );
  roots.push(value);
  await fs.chmod(value, 0o700);
  return value;
}
async function fixture() {
  const dataDir = await root();
  const ledger = await PersistentBackgroundOperationLedger.initialize(
    dataDir,
    owner,
  );
  return {
    dataDir,
    ledger,
    directory: path.join(dataDir, "background-operations"),
    file: path.join(dataDir, "background-operations", "state.json"),
  };
}
async function create(ledger: PersistentBackgroundOperationLedger) {
  await ledger.exclusive(owner, () => ledger.commit(owner, null, snapshot()));
}
afterEach(async () => {
  vi.restoreAllMocks();
  await Promise.all(
    roots
      .splice(0)
      .map((value) => fs.rm(value, { recursive: true, force: true })),
  );
});

it("open does not initialize and initialize does not reset an existing store", async () => {
  const dataDir = await root();
  await expect(
    PersistentBackgroundOperationLedger.open(dataDir, owner),
  ).rejects.toThrow();
  expect(await fs.readdir(dataDir)).toEqual([]);
  await PersistentBackgroundOperationLedger.initialize(dataDir, owner);
  await expect(
    PersistentBackgroundOperationLedger.initialize(dataDir, owner),
  ).rejects.toThrow();
  await fs.unlink(path.join(dataDir, "background-operations", "owner.json"));
  await expect(
    PersistentBackgroundOperationLedger.open(dataDir, owner),
  ).rejects.toThrow();
});

it("persists owner and CAS state privately and reopens without writes", async () => {
  const f = await fixture();
  expect(await f.ledger.load(owner)).toBeNull();
  expect(await fs.readdir(f.directory)).toEqual(["owner.json"]);
  await create(f.ledger);
  expect((await fs.stat(f.directory)).mode & 0o7777).toBe(0o700);
  for (const name of ["owner.json", "state.json"]) {
    const info = await fs.lstat(path.join(f.directory, name));
    expect(info.mode & 0o7777).toBe(0o600);
    expect(info.nlink).toBe(1);
  }
  const before = await fs.readFile(f.file, "utf8");
  const reopened = await PersistentBackgroundOperationLedger.open(
    f.dataDir,
    owner,
  );
  expect(await reopened.load(owner)).toEqual(snapshot());
  expect(await fs.readFile(f.file, "utf8")).toBe(before);
  expect((await fs.readdir(f.directory)).sort()).toEqual([
    "owner.json",
    "state.json",
  ]);
});

it("requires the stable owner and an active exclusive lease for commits", async () => {
  const f = await fixture();
  const wrong = { ...owner, runtimeId: "process-instance-is-not-owner" };
  await expect(f.ledger.commit(owner, null, snapshot())).rejects.toThrow(
    "lock_required",
  );
  await expect(f.ledger.load(wrong)).rejects.toThrow();
  await expect(f.ledger.exclusive(wrong, async () => {})).rejects.toThrow();
  await expect(
    PersistentBackgroundOperationLedger.open(f.dataDir, wrong),
  ).rejects.toThrow();
  await create(f.ledger);
  await f.ledger.exclusive(owner, async () => {
    await expect(f.ledger.commit(owner, null, snapshot())).rejects.toThrow(
      "revision_conflict",
    );
    await f.ledger.commit(owner, 1, snapshot(2, "applying"));
    await expect(
      f.ledger.commit(owner, 1, snapshot(2, "applying")),
    ).rejects.toThrow("revision_conflict");
  });
});

it("delegates invalid metadata and unsafe transition rejection to the frozen model", async () => {
  const f = await fixture();
  await f.ledger.exclusive(owner, async () => {
    await expect(
      f.ledger.commit(owner, null, {
        ...snapshot(),
        secret: "DO_NOT_PERSIST",
      } as BackgroundOperationSnapshot),
    ).rejects.toThrow();
    await f.ledger.commit(owner, null, snapshot());
    await expect(
      f.ledger.commit(owner, 1, snapshot(2, "committed")),
    ).rejects.toThrow();
    await f.ledger.commit(owner, 1, snapshot(2, "applying"));
    await f.ledger.commit(owner, 2, snapshot(3, "unknown"));
    await expect(
      f.ledger.commit(owner, 3, snapshot(4, "applying")),
    ).rejects.toThrow();
  });
  expect((await f.ledger.load(owner))?.operations[0].state).toBe("unknown");
  expect(await fs.readFile(f.file, "utf8")).not.toContain("DO_NOT_PERSIST");
});

it("cannot append past an unresolved operation or silently change its intent", async () => {
  const f = await fixture();
  await create(f.ledger);
  await f.ledger.exclusive(owner, async () => {
    const first = snapshot().operations[0];
    await expect(
      f.ledger.commit(owner, 1, {
        ...snapshot(2),
        operations: [
          first,
          {
            ...first,
            operationId: "f60fc8eb-8e2e-4944-b177-38d58698d42e",
            expectedRevision: 1,
          },
        ],
      }),
    ).rejects.toThrow();
    await expect(
      f.ledger.commit(owner, 1, {
        ...snapshot(2, "applying"),
        operations: [
          { ...first, state: "applying", artifactDigest: "c".repeat(64) },
        ],
      }),
    ).rejects.toThrow();
  });
});

it("serializes concurrent CAS writes within one lease", async () => {
  const f = await fixture();
  await create(f.ledger);
  await f.ledger.exclusive(owner, async () => {
    const results = await Promise.allSettled([
      f.ledger.commit(owner, 1, snapshot(2, "applying")),
      f.ledger.commit(owner, 1, snapshot(2, "applying")),
    ]);
    expect(results.map((result) => result.status).sort()).toEqual([
      "fulfilled",
      "rejected",
    ]);
  });
  expect((await f.ledger.load(owner))?.revision).toBe(2);
});

it("retains the latest 32 operations and cannot evict the unresolved tail", async () => {
  const f = await fixture();
  let current: BackgroundOperationSnapshot | null = null;
  await f.ledger.exclusive(owner, async () => {
    for (let index = 0; index < 33; index++) {
      const previous: BackgroundOperationSnapshot | null = current;
      const revision: number = previous?.revision ?? 0;
      const next: BackgroundOperationSnapshot = {
        ...snapshot(revision + 1),
        operations: [
          ...(previous?.operations.slice(-31) ?? []),
          {
            ...snapshot().operations[0],
            operationId: randomUUID(),
            expectedRevision: revision,
          },
        ],
      };
      await f.ledger.commit(owner, previous?.revision ?? null, next);
      current = next;
      if (index < 32) {
        current = {
          ...next,
          revision: next.revision + 1,
          operations: next.operations.map((entry, i) =>
            i === next.operations.length - 1
              ? { ...entry, state: "rejected" as const }
              : entry,
          ),
        };
        await f.ledger.commit(owner, next.revision, current);
      }
    }
  });
  const actual = await f.ledger.load(owner);
  expect(actual?.operations).toHaveLength(32);
  expect(actual?.operations[0].expectedRevision).toBe(2);
  expect(actual?.operations.at(-1)?.state).toBe("prepared");
  if (!actual) throw new Error("Missing state");
  await f.ledger.exclusive(owner, async () => {
    await expect(
      f.ledger.commit(owner, actual.revision, {
        ...actual,
        revision: actual.revision + 1,
        operations: [
          ...actual.operations.slice(1, -1),
          {
            ...snapshot().operations[0],
            operationId: randomUUID(),
            expectedRevision: actual.revision,
          },
        ],
      }),
    ).rejects.toThrow();
  });
  expect(await f.ledger.load(owner)).toEqual(actual);
});

it("rejects unsafe or symlink data roots without creating a ledger", async () => {
  const dataDir = await root();
  await fs.chmod(dataDir, 0o755);
  await expect(
    PersistentBackgroundOperationLedger.initialize(dataDir, owner),
  ).rejects.toThrow("directory_unsafe");
  expect(await fs.readdir(dataDir)).toEqual([]);
  await fs.chmod(dataDir, 0o700);
  const container = await root();
  const alias = path.join(container, "alias");
  await fs.symlink(dataDir, alias);
  await expect(
    PersistentBackgroundOperationLedger.initialize(alias, owner),
  ).rejects.toThrow("directory_unsafe");
  expect(await fs.readdir(dataDir)).toEqual([]);
});

it("snapshots input before awaits and rejects detached writes after the callback ends", async () => {
  const f = await fixture();
  let detached!: () => Promise<void>;
  await f.ledger.exclusive(owner, async () => {
    const value = snapshot();
    const pending = f.ledger.commit(owner, null, value);
    value.operations[0].planDigest = "c".repeat(64);
    await pending;
    detached = () => f.ledger.commit(owner, 1, snapshot(2, "applying"));
  });
  expect(await f.ledger.load(owner)).toEqual(snapshot());
  await expect(detached()).rejects.toThrow("lock_required");
});

it("excludes a second handle and releases a clean lease on callback failure", async () => {
  const f = await fixture();
  const other = await PersistentBackgroundOperationLedger.open(
    f.dataDir,
    owner,
  );
  await expect(
    f.ledger.exclusive(owner, async () => {
      await expect(other.exclusive(owner, async () => {})).rejects.toThrow(
        "recovery_required",
      );
      throw new Error("synthetic callback failure");
    }),
  ).rejects.toThrow("synthetic callback failure");
  await other.exclusive(owner, async () => {});
});

it("never takes over a stale or malformed lock", async () => {
  const f = await fixture();
  const lock = path.join(f.directory, "execution.lock");
  await fs.mkdir(lock, { mode: 0o700 });
  const text = JSON.stringify({
    ...owner,
    pid: 2147483647,
    nonce: "dead-process",
  });
  await fs.writeFile(path.join(lock, "owner.json"), text, { mode: 0o600 });
  await expect(f.ledger.exclusive(owner, async () => {})).rejects.toThrow(
    "recovery_required",
  );
  expect(await fs.readFile(path.join(lock, "owner.json"), "utf8")).toBe(text);
});

it.each(["owner", "root", "directory"])(
  "rejects replaced %s identity",
  async (kind) => {
    const f = await fixture();
    if (kind === "owner")
      await fs.writeFile(path.join(f.directory, "owner.json"), "{}", {
        mode: 0o600,
      });
    else if (kind === "directory") {
      await fs.rename(f.directory, `${f.directory}.old`);
      await fs.mkdir(f.directory, { mode: 0o700 });
      await fs.copyFile(
        path.join(`${f.directory}.old`, "owner.json"),
        path.join(f.directory, "owner.json"),
      );
    } else {
      await fs.rename(f.dataDir, `${f.dataDir}.old`);
      roots.push(`${f.dataDir}.old`);
      await fs.mkdir(f.dataDir, { mode: 0o700 });
      await fs.mkdir(f.directory, { mode: 0o700 });
      await fs.copyFile(
        path.join(`${f.dataDir}.old`, "background-operations", "owner.json"),
        path.join(f.directory, "owner.json"),
      );
    }
    await expect(f.ledger.load(owner)).rejects.toThrow();
    if (kind === "root")
      await expect(
        PersistentBackgroundOperationLedger.open(f.dataDir, owner),
      ).rejects.toThrow();
  },
);

it.each(["corrupt", "oversize", "symlink", "hardlink", "permissions"])(
  "refuses %s state without repair",
  async (kind) => {
    const f = await fixture();
    await create(f.ledger);
    if (kind === "corrupt") await fs.writeFile(f.file, "{bad");
    if (kind === "oversize")
      await fs.writeFile(f.file, " ".repeat(256 * 1024 + 1));
    if (kind === "permissions") await fs.chmod(f.file, 0o644);
    if (kind === "hardlink") await fs.link(f.file, `${f.file}.alias`);
    if (kind === "symlink") {
      await fs.rename(f.file, `${f.file}.old`);
      await fs.symlink(`${f.file}.old`, f.file);
    }
    await expect(f.ledger.load(owner)).rejects.toThrow();
    expect(await fs.lstat(f.file)).toBeTruthy();
  },
);

it("initial publication never replaces a competing state", async () => {
  const f = await fixture();
  const real = await vi.importActual<typeof fs>("node:fs/promises");
  vi.mocked(fs.link).mockImplementationOnce(async (from, to) => {
    await real.writeFile(to, "competitor", { flag: "wx", mode: 0o600 });
    await real.link(from, to);
  });
  await expect(create(f.ledger)).rejects.toThrow();
  expect(await fs.readFile(f.file, "utf8")).toBe("competitor");
});

it("pre-publication failure preserves the prior state and releases the lock", async () => {
  const f = await fixture();
  await create(f.ledger);
  vi.mocked(fs.rename).mockRejectedValueOnce(
    new Error("synthetic rename failure"),
  );
  await expect(
    f.ledger.exclusive(owner, () =>
      f.ledger.commit(owner, 1, snapshot(2, "applying")),
    ),
  ).rejects.toThrow();
  expect(await f.ledger.load(owner)).toEqual(snapshot());
  await f.ledger.exclusive(owner, async () => {});
});

it("post-publication fsync failure preserves state and poisons the lease across reopen", async () => {
  const f = await fixture();
  await create(f.ledger);
  const real = await vi.importActual<typeof fs>("node:fs/promises");
  vi.mocked(fs.open).mockImplementation(async (...args) => {
    const handle = await real.open(...args);
    if (args[0] === f.directory) {
      const sync = handle.sync.bind(handle);
      handle.sync = async () => {
        if (JSON.parse(await real.readFile(f.file, "utf8")).revision === 2)
          throw new Error("synthetic fsync failure");
        await sync();
      };
    }
    return handle;
  });
  await expect(
    f.ledger.exclusive(owner, () =>
      f.ledger.commit(owner, 1, snapshot(2, "applying")),
    ),
  ).rejects.toThrow("recovery_required");
  vi.mocked(fs.open).mockImplementation(real.open);
  const reopened = await PersistentBackgroundOperationLedger.open(
    f.dataDir,
    owner,
  );
  expect((await reopened.load(owner))?.revision).toBe(2);
  await expect(reopened.exclusive(owner, async () => {})).rejects.toThrow(
    "recovery_required",
  );
});

it("a separate Node process cannot enter the same data-directory lease", async () => {
  const f = await fixture();
  const source = path.dirname(fileURLToPath(import.meta.url));
  // Temporary source copies allow Node's TS loader to resolve the model import
  // without changing the repository or installing a loader.
  const module = path.join(f.dataDir, "ledger.ts");
  const text = await fs.readFile(
    path.join(source, "background-operation-ledger.ts"),
    "utf8",
  );
  await fs.writeFile(
    module,
    text.replaceAll('"./background-operation.js"', '"./model.ts"'),
  );
  await fs.copyFile(
    path.join(source, "background-operation.ts"),
    path.join(f.dataDir, "model.ts"),
  );
  await f.ledger.exclusive(owner, async () => {
    const child = spawn(
      process.execPath,
      [
        "--experimental-transform-types",
        "--input-type=module",
        "-e",
        `
      import {PersistentBackgroundOperationLedger} from ${JSON.stringify(module)};
      const owner=${JSON.stringify(owner)};
      const ledger=await PersistentBackgroundOperationLedger.open(${JSON.stringify(f.dataDir)},owner);
      try { await ledger.exclusive(owner,async()=>console.log('BAD_EFFECT')); }
      catch(error) { console.log(error.message); }
    `,
      ],
      { stdio: ["ignore", "pipe", "pipe"] },
    );
    let out = "",
      err = "";
    child.stdout.on("data", (data) => {
      out += data;
    });
    child.stderr.on("data", (data) => {
      err += data;
    });
    const code = await new Promise<number | null>((resolve, reject) => {
      child.once("exit", resolve);
      child.once("error", reject);
    });
    expect(code, err).toBe(0);
    expect(out.trim()).toBe("background_operation_ledger_recovery_required");
  });
});