Beautiful, AI-native markdown IDE and LLM wiki
TypeScript 2nd-brainagent-skillsclaudecodexcompany-braindocsknowledge-knowledge-baseknowledge-managementllmllm-wikillm-wiki-karpathymarkdownmarkdown-editormdmdxnotespkmsecond-brainskills
import { mkdir, readFile, rename, writeFile } from 'node:fs/promises';
import { dirname } from 'node:path';
const SCHEMA_VERSION = 1 as const;
interface CheckpointEntry<TOutput> {
readonly key: string;
readonly output: TOutput;
readonly recordedAt: string;
}
interface CheckpointFile<TOutput> {
readonly schemaVersion: typeof SCHEMA_VERSION;
readonly entries: ReadonlyArray<CheckpointEntry<TOutput>>;
}
export interface WithCheckpointConfig<TInput> {
readonly checkpointPath: string;
readonly keyOf: (input: TInput) => string;
readonly flushAfterEach: boolean;
}
export async function withCheckpoint<TInput, TOutput>(
operation: (input: TInput) => Promise<TOutput>,
inputs: ReadonlyArray<TInput>,
config: WithCheckpointConfig<TInput>,
): Promise<ReadonlyArray<TOutput>> {
if (inputs.length === 0) return [];
const inputKeysInOrder: string[] = inputs.map(config.keyOf);
const duplicates = findDuplicateKeys(inputKeysInOrder);
if (duplicates.length > 0) {
throw new Error(
`withCheckpoint: keyOf produced duplicate keys for distinct inputs ` +
`(programmer error — inputs must map 1:1 to keys). ` +
`Duplicates: ${duplicates.join(', ')}`,
);
}
const completed = await readCheckpoint<TOutput>(config.checkpointPath);
const outputs: TOutput[] = [];
let pendingError: unknown = null;
for (let i = 0; i < inputs.length; i++) {
const input = inputs[i] as TInput;
const key = inputKeysInOrder[i] as string;
const prior = completed.get(key);
if (prior !== undefined) {
outputs.push(prior.output);
continue;
}
let result: TOutput;
try {
result = await operation(input);
} catch (err) {
pendingError = err;
break;
}
completed.set(key, { output: result, recordedAt: new Date().toISOString() });
outputs.push(result);
if (config.flushAfterEach) {
await flushCheckpoint(config.checkpointPath, inputKeysInOrder, completed);
}
}
if (!config.flushAfterEach) {
try {
await flushCheckpoint(config.checkpointPath, inputKeysInOrder, completed);
} catch (flushErr) {
if (pendingError !== null) {
throw new AggregateError(
[pendingError, flushErr],
'withCheckpoint: operation failed AND final flush failed. Prior successes may not be durable. Inspect .errors for both causes.',
);
}
throw flushErr;
}
}
if (pendingError !== null) throw pendingError;
return outputs;
}
interface InMemoryEntry<TOutput> {
readonly output: TOutput;
readonly recordedAt: string;
}
function findDuplicateKeys(keys: ReadonlyArray<string>): ReadonlyArray<string> {
const seen = new Set<string>();
const duplicates: string[] = [];
for (const k of keys) {
if (seen.has(k)) duplicates.push(k);
seen.add(k);
}
return duplicates;
}
async function readCheckpoint<TOutput>(path: string): Promise<Map<string, InMemoryEntry<TOutput>>> {
let raw: string;
try {
raw = await readFile(path, 'utf8');
} catch (err) {
if (isNoEnt(err)) return new Map();
throw new Error(
`withCheckpoint: failed to read checkpoint at ${path}: ` +
`${err instanceof Error ? err.message : String(err)}`,
);
}
let parsed: unknown;
try {
parsed = JSON.parse(raw);
} catch (err) {
throw new Error(
`withCheckpoint: checkpoint at ${path} is not valid JSON ` +
`(corrupted? delete the file to start fresh). Parse error: ` +
`${err instanceof Error ? err.message : String(err)}`,
);
}
return validateCheckpoint<TOutput>(parsed, path);
}
function validateCheckpoint<TOutput>(
parsed: unknown,
path: string,
): Map<string, InMemoryEntry<TOutput>> {
if (parsed === null || typeof parsed !== 'object') {
throw new Error(
`withCheckpoint: checkpoint at ${path} is not a JSON object ` +
`(corrupted? delete the file to start fresh).`,
);
}
if (!('schemaVersion' in parsed)) {
throw new Error(
`withCheckpoint: checkpoint at ${path} is missing 'schemaVersion' field ` +
`(corrupted or pre-versioned? delete the file to start fresh).`,
);
}
const schemaVersion = (parsed as { schemaVersion: unknown }).schemaVersion;
if (schemaVersion !== SCHEMA_VERSION) {
throw new Error(
`withCheckpoint: checkpoint at ${path} has schemaVersion ` +
`${String(schemaVersion)}, expected ${SCHEMA_VERSION}. ` +
`The checkpoint format has changed; delete the file to start fresh.`,
);
}
if (!('entries' in parsed)) {
throw new Error(
`withCheckpoint: checkpoint at ${path} is missing 'entries' field ` +
`(corrupted? delete the file to start fresh).`,
);
}
const entries = (parsed as { entries: unknown }).entries;
if (!Array.isArray(entries)) {
throw new Error(
`withCheckpoint: checkpoint at ${path} has non-array 'entries' ` +
`(corrupted? delete the file to start fresh).`,
);
}
const map = new Map<string, InMemoryEntry<TOutput>>();
for (const entry of entries) {
if (entry === null || typeof entry !== 'object') {
throw new Error(
`withCheckpoint: checkpoint at ${path} contains a non-object entry ` +
`(corrupted? delete the file to start fresh).`,
);
}
if (!('key' in entry) || !('output' in entry) || !('recordedAt' in entry)) {
throw new Error(
`withCheckpoint: checkpoint at ${path} contains an entry missing ` +
`key/output/recordedAt (corrupted? delete the file to start fresh).`,
);
}
const { key, output, recordedAt } = entry as {
key: unknown;
output: unknown;
recordedAt: unknown;
};
if (typeof key !== 'string') {
throw new Error(
`withCheckpoint: checkpoint at ${path} contains a non-string key ` +
`(corrupted? delete the file to start fresh).`,
);
}
if (typeof recordedAt !== 'string') {
throw new Error(
`withCheckpoint: checkpoint at ${path} contains a non-string recordedAt ` +
`(corrupted? delete the file to start fresh).`,
);
}
map.set(key, { output: output as TOutput, recordedAt });
}
return map;
}
async function flushCheckpoint<TOutput>(
path: string,
inputKeysInOrder: ReadonlyArray<string>,
completed: ReadonlyMap<string, InMemoryEntry<TOutput>>,
): Promise<void> {
const entries: Array<CheckpointEntry<TOutput>> = [];
for (const key of inputKeysInOrder) {
const e = completed.get(key);
if (e !== undefined) {
entries.push({ key, output: e.output, recordedAt: e.recordedAt });
}
}
const file: CheckpointFile<TOutput> = {
schemaVersion: SCHEMA_VERSION,
entries,
};
const tmpPath = `${path}.tmp`;
await mkdir(dirname(path), { recursive: true });
await writeFile(tmpPath, `${JSON.stringify(file, null, 2)}\n`, 'utf8');
await rename(tmpPath, path);
}
function isNoEnt(err: unknown): boolean {
return (
typeof err === 'object' &&
err !== null &&
'code' in err &&
(err as { code?: unknown }).code === 'ENOENT'
);
}