Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, vi, beforeAll, beforeEach, afterEach, afterAll } from 'vitest';
import { readFile, writeFile, rm, mkdir } from 'fs/promises';
import { mkdtempSync, rmSync, writeFileSync, readFileSync, readdirSync } from 'fs';
import { join } from 'path';
import { tmpdir, homedir } from 'os';
import { createHash } from 'crypto';
import { pinPlatform } from './testHelper.js';
import * as fsPromises from 'fs/promises';
// Mock fs/promises so the `ensureDir` regression test can force `mkdir` to
// throw a spurious Windows error, and the readJSONFileStrict tests can force a
// specific `readFile` errno (EACCES can't be provoked portably — chmod 000 is a
// no-op for root, which is how the container CI runs). Both default to delegating
// to the real implementation (so every other test in this file is unaffected);
// individual tests override with `mockRejectedValueOnce`.
// `rename` and `readdir` are wrapped for the same reason (#4095): the Windows
// swap-window retries key off transient EPERM/EACCES/EBUSY/ENOENT errnos that
// cannot be provoked on a POSIX test host.
vi.mock('fs/promises', async (importOriginal) => {
const actual = await importOriginal();
return {
...actual,
mkdir: vi.fn((...args) => actual.mkdir(...args)),
open: vi.fn((...args) => actual.open(...args)),
readFile: vi.fn((...args) => actual.readFile(...args)),
rename: vi.fn((...args) => actual.rename(...args)),
writeFile: vi.fn((...args) => actual.writeFile(...args)),
chmod: vi.fn((...args) => actual.chmod(...args)),
readdir: vi.fn((...args) => actual.readdir(...args)),
stat: vi.fn((...args) => actual.stat(...args)),
};
});
import {
assertSafeFilename,
isSafeFilename,
atomicWrite,
ensureDir,
pathExists,
expandHome,
listDirectoryByExtension,
safeJSONParse,
safeJSONLParse,
createCachedStore,
readJSONFile,
readJSONFileStrict,
tryReadFile,
tryReadFileStrict,
readJSONLFile,
appendJSONLine,
readJSONLines,
writeJSONLines,
formatDuration,
readFileTail,
dirSize,
sha256File,
resolveImageInputPath,
resolveScreenshot,
isPathInsideDir,
PATHS,
sanitizeFilename,
getFileExtension,
getMimeType,
detectImageFormat,
EXTENSION_MIME_MAP,
ATTACHMENT_ALLOWED_EXTENSIONS,
SONGBOOK_ATTACHMENT_EXTENSIONS,
saveBase64Upload,
saveImageUpload,
importFileToDir,
serveLocalFile,
watchForFile,
} from './fileUtils.js';
import { copyFileSync, existsSync, mkdirSync } from 'fs';
import { dirname } from 'path';
import { fileURLToPath } from 'url';
const __dirname_test = dirname(fileURLToPath(import.meta.url));
// The unmocked fs/promises, used to restore the delegating mock implementations
// after a test installs a persistent one.
let realFsPromises;
beforeAll(async () => {
realFsPromises = await vi.importActual('fs/promises');
});
describe('importFileToDir', () => {
let dir;
let source;
beforeEach(async () => {
dir = mkdtempSync(join(tmpdir(), 'import-file-test-'));
source = join(dir, 'source.bin');
await writeFile(source, 'fixture');
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it('rejects a persisted extension outside an explicit allowlist', async () => {
await expect(importFileToDir(source, 'payload.svg', join(dir, 'dest'), { extensions: ['.png'] }))
.rejects.toMatchObject({ status: 400, code: 'INVALID_FILE_TYPE' });
});
});
const restoreFsMocks = () => {
for (const name of ['readFile', 'rename', 'readdir', 'mkdir', 'stat', 'writeFile', 'chmod', 'open']) {
fsPromises[name].mockReset();
fsPromises[name].mockImplementation((...args) => realFsPromises[name](...args));
}
};
/**
* Run `fn` with EVERY `readFile` attempt rejected by `err`, then restore the
* delegating mock.
*
* A single `mockRejectedValueOnce` is not enough since #4095: on win32
* `readJSONFileStrict` retries a transient lock, so the first attempt would
* consume the queued rejection and the retry would read the real file. A test
* pinning "an unreadable file is not an empty one" has to keep it unreadable —
* which is also the contract that actually survives the retries.
*/
async function withUnreadableFile(err, fn) {
fsPromises.readFile.mockImplementation(() => Promise.reject(err));
try {
return await fn();
} finally {
restoreFsMocks();
}
}
describe('fileUtils', () => {
describe('safeJSONParse', () => {
it('should parse valid JSON object', () => {
const result = safeJSONParse('{"key": "value"}', {});
expect(result).toEqual({ key: 'value' });
});
it('should parse valid JSON array', () => {
const result = safeJSONParse('[1, 2, 3]', []);
expect(result).toEqual([1, 2, 3]);
});
it('should return default value for empty string', () => {
const result = safeJSONParse('', { default: true });
expect(result).toEqual({ default: true });
});
it('should return default value for null input', () => {
const result = safeJSONParse(null, []);
expect(result).toEqual([]);
});
it('should return default value for invalid JSON', () => {
const result = safeJSONParse('not json', { fallback: 'value' });
expect(result).toEqual({ fallback: 'value' });
});
it('should return default value for JSON with trailing comma', () => {
const result = safeJSONParse('{"a": 1,}', {});
expect(result).toEqual({});
});
it('should return default value for truncated JSON', () => {
const result = safeJSONParse('{"key": "value', {});
expect(result).toEqual({});
});
it('should return null as default when no defaultValue provided', () => {
const result = safeJSONParse('invalid');
expect(result).toBe(null);
});
it('should reject arrays when allowArray is false', () => {
const result = safeJSONParse('[1, 2, 3]', {}, { allowArray: false });
expect(result).toEqual({});
});
it('should still reject a root array under allowArray: false when defaultValue is not an array', () => {
const result = safeJSONParse('["a", "b"]', { fallback: true }, { allowArray: false });
expect(result).toEqual({ fallback: true });
});
it('should parse a top-level scalar string', () => {
expect(safeJSONParse('"hello"', null)).toBe('hello');
});
it('should parse a top-level scalar number', () => {
expect(safeJSONParse('123', null)).toBe(123);
});
it('should parse a top-level scalar boolean', () => {
expect(safeJSONParse('true', null)).toBe(true);
});
it('should log warning when logError is true', () => {
const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
safeJSONParse('invalid', {}, { logError: true });
expect(consoleSpy).toHaveBeenCalled();
consoleSpy.mockRestore();
});
it('should include context in log message', () => {
const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
safeJSONParse('invalid', {}, { logError: true, context: 'test-file.json' });
expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('test-file.json'));
consoleSpy.mockRestore();
});
it('should not log for empty input even with logError true', () => {
const consoleSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
safeJSONParse('', {}, { logError: true });
expect(consoleSpy).not.toHaveBeenCalled();
consoleSpy.mockRestore();
});
it('should handle syntax error in structurally valid JSON', () => {
const consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
const result = safeJSONParse('{"key": undefined}', { fallback: true }, { logError: true });
expect(result).toEqual({ fallback: true });
expect(consoleWarnSpy).toHaveBeenCalled();
consoleWarnSpy.mockRestore();
});
});
describe('safeJSONLParse', () => {
it('should parse valid JSONL content', () => {
const content = '{"a": 1}\n{"b": 2}\n{"c": 3}';
const result = safeJSONLParse(content);
expect(result).toEqual([{ a: 1 }, { b: 2 }, { c: 3 }]);
});
it('should skip empty lines', () => {
const content = '{"a": 1}\n\n{"b": 2}\n';
const result = safeJSONLParse(content);
expect(result).toEqual([{ a: 1 }, { b: 2 }]);
});
it('should skip whitespace-only lines', () => {
const content = '{"a": 1}\n \n{"b": 2}';
const result = safeJSONLParse(content);
expect(result).toEqual([{ a: 1 }, { b: 2 }]);
});
it('should skip invalid lines and continue parsing', () => {
const content = '{"a": 1}\ninvalid json\n{"b": 2}';
const result = safeJSONLParse(content);
expect(result).toEqual([{ a: 1 }, { b: 2 }]);
});
it('should return empty array for empty content', () => {
expect(safeJSONLParse('')).toEqual([]);
});
it('should return empty array for null content', () => {
expect(safeJSONLParse(null)).toEqual([]);
});
it('should return empty array for whitespace-only content', () => {
expect(safeJSONLParse(' \n ')).toEqual([]);
});
it('should handle single line without trailing newline', () => {
const result = safeJSONLParse('{"single": "line"}');
expect(result).toEqual([{ single: 'line' }]);
});
it('should reject array values in lines (JSONL expects objects)', () => {
const content = '{"a": 1}\n[1, 2, 3]\n{"b": 2}';
const result = safeJSONLParse(content);
// Arrays are rejected because allowArray: false is passed internally
expect(result).toEqual([{ a: 1 }, { b: 2 }]);
});
it('should handle lines with only truncated JSON', () => {
const content = '{"complete": true}\n{"incomplete":';
const result = safeJSONLParse(content);
expect(result).toEqual([{ complete: true }]);
});
it('should handle CRLF line endings (Windows)', () => {
const content = '{"a": 1}\r\n{"b": 2}\r\n{"c": 3}';
const result = safeJSONLParse(content);
expect(result).toEqual([{ a: 1 }, { b: 2 }, { c: 3 }]);
});
it('should handle mixed LF and CRLF line endings', () => {
const content = '{"a": 1}\n{"b": 2}\r\n{"c": 3}';
const result = safeJSONLParse(content);
expect(result).toEqual([{ a: 1 }, { b: 2 }, { c: 3 }]);
});
});
describe('readJSONFile', () => {
const testDir = join(tmpdir(), 'fileutils-test-' + Date.now());
beforeEach(async () => {
await mkdir(testDir, { recursive: true });
});
afterEach(async () => {
await rm(testDir, { recursive: true, force: true });
});
it('should read and parse valid JSON file', async () => {
const filePath = join(testDir, 'valid.json');
await writeFile(filePath, '{"key": "value"}');
const result = await readJSONFile(filePath, {});
expect(result).toEqual({ key: 'value' });
});
it('should return default value for non-existent file', async () => {
const result = await readJSONFile('/nonexistent/path.json', { default: true });
expect(result).toEqual({ default: true });
});
it('should return default value for empty file', async () => {
const filePath = join(testDir, 'empty.json');
await writeFile(filePath, '');
const result = await readJSONFile(filePath, { empty: true });
expect(result).toEqual({ empty: true });
});
it('should return default value for corrupted file', async () => {
const filePath = join(testDir, 'corrupted.json');
await writeFile(filePath, '{"incomplete":');
const result = await readJSONFile(filePath, { fallback: true });
expect(result).toEqual({ fallback: true });
});
it('should handle arrays when allowArray is true', async () => {
const filePath = join(testDir, 'array.json');
await writeFile(filePath, '[1, 2, 3]');
const result = await readJSONFile(filePath, []);
expect(result).toEqual([1, 2, 3]);
});
it('should reject arrays when allowArray is false', async () => {
const filePath = join(testDir, 'array.json');
await writeFile(filePath, '[1, 2, 3]');
const result = await readJSONFile(filePath, {}, { allowArray: false });
expect(result).toEqual({});
});
it('returns the array default when an array-defaulted file holds an object root', async () => {
// Regression guard: every array-defaulted caller in-tree goes straight to
// .filter/.find on the result (services/review.js#getItems,
// services/videoGen/local.js, routes/videoGen.js), so handing back a parsed
// object root would TypeError a request that used to degrade to an empty list.
const filePath = join(testDir, 'object-root.json');
await writeFile(filePath, '{"a":1}');
expect(await readJSONFile(filePath, [])).toEqual([]);
});
it('returns the caller’s array default for garbage, not a manufactured []', async () => {
// The noisy-output extraction used to manufacture a literal '[]' from text
// holding no array, so a non-empty array default was silently replaced by an
// empty one. The documented contract is "default value if the file is
// invalid" — every in-tree caller passes `[]`, where the two agree.
const filePath = join(testDir, 'garbage.json');
await writeFile(filePath, 'not json at all');
expect(await readJSONFile(filePath, ['fallback'])).toEqual(['fallback']);
});
it('still extracts an array out of noisy output (pm2 jlist with ANSI codes)', async () => {
// Real ANSI output leads with ESC, not '[' — the extraction is deliberately
// skipped for text already starting with '[' (it would be self-defeating on a
// genuine JSON array), so `\x1b` here is load-bearing, not decoration.
const filePath = join(testDir, 'noisy.json');
await writeFile(filePath, '\x1b[31mwarn\x1b[0m [{"pid":1}]');
expect(await readJSONFile(filePath, [])).toEqual([{ pid: 1 }]);
});
});
// #2726: `readJSONFile` collapses "absent", "unreadable", and "corrupt" into the
// same default, so a caller counting records can't tell a real empty from a failed
// read and reports a fake 0. These pin the distinction the strict variant restores.
describe('readJSONFileStrict', () => {
const testDir = join(tmpdir(), 'fileutils-strict-test-' + Date.now());
let warnSpy;
beforeEach(async () => {
await mkdir(testDir, { recursive: true });
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
});
afterEach(async () => {
warnSpy.mockRestore();
await rm(testDir, { recursive: true, force: true });
});
const eacces = () => Object.assign(new Error('permission denied'), { code: 'EACCES' });
it('reports ENOENT as a TRUSTWORTHY empty — absent is not a failure', async () => {
const result = await readJSONFileStrict(join(testDir, 'never-written.json'), { sessions: [] });
expect(result).toEqual({ ok: true, value: { sessions: [] } });
});
it('does not log for a genuinely absent file', async () => {
await readJSONFileStrict(join(testDir, 'never-written.json'), []);
expect(warnSpy).not.toHaveBeenCalled();
});
it('reports EACCES as NOT ok — an unreadable file is not an empty one', async () => {
const filePath = join(testDir, 'locked.json');
await writeFile(filePath, '{"sessions": [{"id": "s1"}]}');
const result = await withUnreadableFile(eacces(), () => readJSONFileStrict(filePath, { sessions: [] }));
// `value` still carries the default so an ok-indifferent caller behaves as
// before; `ok: false` is what makes the failure legible.
expect(result).toEqual({ ok: false, value: { sessions: [] } });
});
it('reports a real non-ENOENT errno as NOT ok (reading a directory → EISDIR)', async () => {
// Unmocked, no synthetic errno: proves the classification keys off "not
// ENOENT" rather than a hard-coded list the real world can step outside of.
const result = await readJSONFileStrict(testDir, []);
expect(result.ok).toBe(false);
});
it('reports malformed JSON as NOT ok — corrupt bytes are not an empty collection', async () => {
const filePath = join(testDir, 'corrupt.json');
await writeFile(filePath, '{"incomplete":');
expect(await readJSONFileStrict(filePath, { fallback: true }))
.toEqual({ ok: false, value: { fallback: true } });
});
it('reports a truncated (empty) file as NOT ok', async () => {
const filePath = join(testDir, 'empty.json');
await writeFile(filePath, '');
expect((await readJSONFileStrict(filePath, [])).ok).toBe(false);
});
it('reports a parsed file as ok with its value', async () => {
const filePath = join(testDir, 'valid.json');
await writeFile(filePath, '{"sessions": [{"id": "s1"}]}');
expect(await readJSONFileStrict(filePath, { sessions: [] }))
.toEqual({ ok: true, value: { sessions: [{ id: 's1' }] } });
});
it('reports a legitimately empty collection as ok — the whole point', async () => {
const filePath = join(testDir, 'empty-list.json');
await writeFile(filePath, '[]');
expect(await readJSONFileStrict(filePath, ['fallback'])).toEqual({ ok: true, value: [] });
});
it('distinguishes a file that legitimately CONTAINS the default from a failed read', async () => {
const filePath = join(testDir, 'same-as-default.json');
await writeFile(filePath, '{"sessions":[]}');
// Both return the same `value`; only `ok` tells them apart — which is why the
// parse sentinel can't be an in-band marker like null.
expect(await readJSONFileStrict(filePath, { sessions: [] })).toEqual({ ok: true, value: { sessions: [] } });
expect(await withUnreadableFile(eacces(), () => readJSONFileStrict(filePath, { sessions: [] })))
.toEqual({ ok: false, value: { sessions: [] } });
});
it('keeps readJSONFile’s noisy-output extraction for array defaults', async () => {
// safeJSONParse keys extraction off `Array.isArray(defaultValue)`, and the
// strict path passes a non-array sentinel as its fallback — so this pins that
// the extraction still runs against the caller's real default (pm2 jlist).
const filePath = join(testDir, 'noisy.json');
await writeFile(filePath, '\x1b[31mwarning\x1b[0m [{"id":1}]');
expect(await readJSONFileStrict(filePath, [])).toEqual({ ok: true, value: [{ id: 1 }] });
});
it('reports an object root as NOT ok when the caller declared an array default', async () => {
// The array default is a declared shape expectation — a strict caller counting
// a list must refuse an object it cannot count, rather than trusting it.
const filePath = join(testDir, 'object-root.json');
await writeFile(filePath, '{"sessions":[1,2]}');
expect(await readJSONFileStrict(filePath, [])).toEqual({ ok: false, value: [] });
});
it('does NOT manufacture a trustworthy empty from noise holding no array', async () => {
// extractJSONArray returns a literal '[]' when it finds nothing, which parses
// cleanly — an array-defaulted strict read would otherwise report corrupt bytes
// as `ok: true, value: []`, silently un-stricting itself.
const filePath = join(testDir, 'garbage.json');
await writeFile(filePath, 'not json at all');
expect(await readJSONFileStrict(filePath, [])).toEqual({ ok: false, value: [] });
});
it('honours allowArray: false', async () => {
const filePath = join(testDir, 'array.json');
await writeFile(filePath, '[1, 2, 3]');
expect(await readJSONFileStrict(filePath, {}, { allowArray: false }))
.toEqual({ ok: false, value: {} });
});
it('stays silent when logError is false', async () => {
const filePath = join(testDir, 'corrupt.json');
await writeFile(filePath, '{"incomplete":');
await readJSONFileStrict(filePath, {}, { logError: false });
expect(warnSpy).not.toHaveBeenCalled();
});
});
// #4115: `tryReadFile` had the same absent-vs-unreadable conflation as
// `readJSONFile` (a bare `.catch(() => null)`) but no strict counterpart at all.
// These pin the three-state contract for the raw-bytes variant.
describe('tryReadFileStrict', () => {
const testDir = join(tmpdir(), 'fileutils-tryreadstrict-test-' + Date.now());
beforeEach(async () => {
await mkdir(testDir, { recursive: true });
});
afterEach(async () => {
await rm(testDir, { recursive: true, force: true });
});
const eacces = () => Object.assign(new Error('permission denied'), { code: 'EACCES' });
it('reports ENOENT as a TRUSTWORTHY absence', async () => {
expect(await tryReadFileStrict(join(testDir, 'never-written.txt')))
.toEqual({ ok: true, value: null });
});
it('reports EACCES as NOT ok — an unreadable file is not an absent one', async () => {
const filePath = join(testDir, 'locked.txt');
await writeFile(filePath, 'real contents');
expect(await withUnreadableFile(eacces(), () => tryReadFileStrict(filePath)))
.toEqual({ ok: false, value: null });
});
it('reports a real non-ENOENT errno as NOT ok (reading a directory → EISDIR)', async () => {
// Unmocked, no synthetic errno: proves the classification keys off "not
// ENOENT" rather than a hard-coded list the real world can step outside of.
expect((await tryReadFileStrict(testDir)).ok).toBe(false);
});
it('returns the contents with ok when the read succeeds', async () => {
const filePath = join(testDir, 'present.txt');
await writeFile(filePath, 'hello');
expect(await tryReadFileStrict(filePath)).toEqual({ ok: true, value: 'hello' });
});
it('keeps an EMPTY file distinguishable from an absent one', async () => {
const filePath = join(testDir, 'empty.txt');
await writeFile(filePath, '');
expect(await tryReadFileStrict(filePath)).toEqual({ ok: true, value: '' });
expect(await tryReadFileStrict(join(testDir, 'gone.txt'))).toEqual({ ok: true, value: null });
});
it('honours a null encoding and returns a Buffer', async () => {
const filePath = join(testDir, 'bytes.bin');
await writeFile(filePath, Buffer.from([0x00, 0x01, 0xff]));
const { ok, value } = await tryReadFileStrict(filePath, null);
expect(ok).toBe(true);
expect(Buffer.isBuffer(value)).toBe(true);
expect([...value]).toEqual([0x00, 0x01, 0xff]);
});
it('matches tryReadFile on the value for every non-error case', async () => {
const filePath = join(testDir, 'parity.txt');
await writeFile(filePath, 'same bytes');
expect((await tryReadFileStrict(filePath)).value).toBe(await tryReadFile(filePath));
expect((await tryReadFileStrict(join(testDir, 'nope.txt'))).value).toBe(await tryReadFile(join(testDir, 'nope.txt')));
});
});
// The `strict` option is the same classification, surfaced as a throw — the shape
// the Character signal readers need, since a rejection is how every DB-backed
// getter already reports failure (#2726).
describe('readJSONFile with { strict: true }', () => {
const testDir = join(tmpdir(), 'fileutils-strictopt-test-' + Date.now());
let warnSpy;
beforeEach(async () => {
await mkdir(testDir, { recursive: true });
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
});
afterEach(async () => {
warnSpy.mockRestore();
await rm(testDir, { recursive: true, force: true });
});
it('returns the default for an absent file — strict does not mean paranoid', async () => {
const result = await readJSONFile(join(testDir, 'never-written.json'), { sessions: [] }, { strict: true });
expect(result).toEqual({ sessions: [] });
});
it('throws for an unreadable file instead of returning a fake empty', async () => {
const filePath = join(testDir, 'locked.json');
await writeFile(filePath, '{"sessions": []}');
const eacces = Object.assign(new Error('permission denied'), { code: 'EACCES' });
await withUnreadableFile(eacces, () => expect(readJSONFile(filePath, { sessions: [] }, { strict: true }))
.rejects.toThrow(/Unreadable JSON file/));
});
it('throws for corrupt JSON', async () => {
const filePath = join(testDir, 'corrupt.json');
await writeFile(filePath, 'not json at all');
await expect(readJSONFile(filePath, [], { strict: true })).rejects.toMatchObject({
message: expect.stringMatching(/Unreadable JSON file/),
status: 500,
code: 'UNREADABLE_STORE',
responseMessage: 'Durable data store is unreadable',
});
});
it('returns the parsed value when the read succeeds', async () => {
const filePath = join(testDir, 'valid.json');
await writeFile(filePath, '{"sessions":[{"id":"s1"}]}');
expect(await readJSONFile(filePath, { sessions: [] }, { strict: true }))
.toEqual({ sessions: [{ id: 's1' }] });
});
it('leaves every existing (non-strict) caller swallowing exactly as before', async () => {
const filePath = join(testDir, 'corrupt.json');
await writeFile(filePath, '{"incomplete":');
expect(await readJSONFile(filePath, { fallback: true })).toEqual({ fallback: true });
fsPromises.readFile.mockRejectedValueOnce(Object.assign(new Error('nope'), { code: 'EACCES' }));
expect(await readJSONFile(filePath, { fallback: true })).toEqual({ fallback: true });
});
});
describe('readJSONLFile', () => {
const testDir = join(tmpdir(), 'fileutils-jsonl-test-' + Date.now());
beforeEach(async () => {
await mkdir(testDir, { recursive: true });
});
afterEach(async () => {
await rm(testDir, { recursive: true, force: true });
});
it('should read and parse valid JSONL file', async () => {
const filePath = join(testDir, 'valid.jsonl');
await writeFile(filePath, '{"a": 1}\n{"b": 2}\n{"c": 3}');
const result = await readJSONLFile(filePath);
expect(result).toEqual([{ a: 1 }, { b: 2 }, { c: 3 }]);
});
it('should return empty array for non-existent file', async () => {
const result = await readJSONLFile('/nonexistent/path.jsonl');
expect(result).toEqual([]);
});
it('should return empty array for empty file', async () => {
const filePath = join(testDir, 'empty.jsonl');
await writeFile(filePath, '');
const result = await readJSONLFile(filePath);
expect(result).toEqual([]);
});
it('should skip invalid lines in JSONL file', async () => {
const filePath = join(testDir, 'mixed.jsonl');
await writeFile(filePath, '{"valid": 1}\nnot json\n{"also": "valid"}');
const result = await readJSONLFile(filePath);
expect(result).toEqual([{ valid: 1 }, { also: 'valid' }]);
});
});
describe('JSONL write helpers', () => {
let testDir;
beforeEach(async () => {
testDir = mkdtempSync(join(tmpdir(), 'fileutils-jsonl-write-test-'));
});
afterEach(async () => {
await rm(testDir, { recursive: true, force: true });
});
it('appendJSONLine creates parent dirs and appends one record per line', async () => {
const filePath = join(testDir, 'nested', 'events.jsonl');
await appendJSONLine(filePath, { id: 'a', n: 1 });
await appendJSONLine(filePath, { id: 'b', n: 2 });
expect(await readFile(filePath, 'utf-8')).toBe('{"id":"a","n":1}\n{"id":"b","n":2}\n');
expect(await readJSONLines(filePath)).toEqual([{ id: 'a', n: 1 }, { id: 'b', n: 2 }]);
});
it('readJSONLines supports offset and limit', async () => {
const filePath = join(testDir, 'events.jsonl');
await writeFile(filePath, '{"id":"a"}\n{"id":"b"}\n{"id":"c"}\n');
expect(await readJSONLines(filePath, { from: 1, limit: 1 })).toEqual([{ id: 'b' }]);
expect(await readJSONLines(filePath, { from: 2 })).toEqual([{ id: 'c' }]);
expect(await readJSONLines(filePath, { limit: 0 })).toEqual([]);
});
it('writeJSONLines atomically replaces the file', async () => {
const filePath = join(testDir, 'events.jsonl');
await appendJSONLine(filePath, { id: 'old' });
await writeJSONLines(filePath, [{ id: 'new-1' }, { id: 'new-2' }]);
expect(await readFile(filePath, 'utf-8')).toBe('{"id":"new-1"}\n{"id":"new-2"}\n');
expect(await readJSONLines(filePath)).toEqual([{ id: 'new-1' }, { id: 'new-2' }]);
});
it('rejects non-serializable values', async () => {
await expect(appendJSONLine(join(testDir, 'bad.jsonl'), undefined))
.rejects.toThrow(/JSON-serializable/);
await expect(writeJSONLines(join(testDir, 'bad.jsonl'), [undefined]))
.rejects.toThrow(/JSON-serializable/);
});
});
describe('pathExists', () => {
const tmpRoot = join(tmpdir(), `fileutils-pathexists-${process.pid}-${Date.now()}`);
beforeEach(() => mkdir(tmpRoot, { recursive: true }));
afterEach(() => rm(tmpRoot, { recursive: true, force: true }));
it('resolves true for an existing file', async () => {
const f = join(tmpRoot, 'present.txt');
await writeFile(f, 'hi');
expect(await pathExists(f)).toBe(true);
});
it('resolves true for an existing directory', async () => {
expect(await pathExists(tmpRoot)).toBe(true);
});
it('resolves false for a missing path without throwing', async () => {
expect(await pathExists(join(tmpRoot, 'nope.txt'))).toBe(false);
});
});
describe('formatDuration', () => {
it('should return "0m" for zero or falsy values', () => {
expect(formatDuration(0)).toBe('0m');
expect(formatDuration(null)).toBe('0m');
expect(formatDuration(undefined)).toBe('0m');
});
it('should format minutes correctly', () => {
expect(formatDuration(60000)).toBe('1m');
expect(formatDuration(300000)).toBe('5m');
expect(formatDuration(59 * 60000)).toBe('59m');
});
it('should format hours and minutes correctly', () => {
expect(formatDuration(60 * 60000)).toBe('1h 0m');
expect(formatDuration(90 * 60000)).toBe('1h 30m');
expect(formatDuration(150 * 60000)).toBe('2h 30m');
});
it('should format days and hours correctly', () => {
expect(formatDuration(24 * 60 * 60000)).toBe('1d 0h');
expect(formatDuration(25 * 60 * 60000)).toBe('1d 1h');
expect(formatDuration(48 * 60 * 60000)).toBe('2d 0h');
expect(formatDuration(50 * 60 * 60000)).toBe('2d 2h');
});
});
describe('isPathInsideDir', () => {
it('accepts a file directly inside the directory', () => {
expect(isPathInsideDir('/data/uploads', '/data/uploads/foo.png')).toBe(true);
expect(isPathInsideDir('/data/uploads', '/data/uploads/sub/foo.png')).toBe(true);
});
it('rejects a traversal that escapes the directory', () => {
expect(isPathInsideDir('/data/uploads', '/data/uploads/../etc/passwd')).toBe(false);
expect(isPathInsideDir('/data/uploads', '/etc/passwd')).toBe(false);
});
// Windows and macOS filesystems are case-insensitive, so two spellings of
// one directory are one directory. A case-SENSITIVE prefix compare reported
// a managed worktree as `worktree-unmanaged-location` and the reaper
// silently never reaped it — git reports paths its own way (drive-letter
// case, 8.3 short names), which need not match how PortOS spelled the root.
it('matches case-insensitively only where the filesystem is', () => {
const insensitive = process.platform === 'win32' || process.platform === 'darwin';
expect(isPathInsideDir('/data/Uploads', '/data/uploads/foo.png')).toBe(insensitive);
expect(isPathInsideDir('/data/uploads', '/data/UPLOADS/sub/foo.png')).toBe(insensitive);
});
// Case-folding must only ever recognize the SAME directory — never widen
// containment to a sibling or admit a traversal.
it('still rejects escapes and prefix-sibling directories under case-folding', () => {
expect(isPathInsideDir('/data/Uploads', '/data/uploads-evil/foo.png')).toBe(false);
expect(isPathInsideDir('/data/Uploads', '/data/UPLOADS/../etc/passwd')).toBe(false);
expect(isPathInsideDir('/data/Uploads', '/etc/passwd')).toBe(false);
// The root itself is not "inside" itself, in either casing.
expect(isPathInsideDir('/data/Uploads', '/data/uploads')).toBe(false);
});
it('rejects a sibling dir whose name merely starts with the root (the trailing-sep bug)', () => {
// The old `startsWith(DIR)` check without a trailing separator let
// `/data/uploads-evil/x` slip through because the string prefix matched.
expect(isPathInsideDir('/data/uploads', '/data/uploads-evil/x.png')).toBe(false);
});
it('rejects the root itself (no trailing separator on the bare root)', () => {
expect(isPathInsideDir('/data/uploads', '/data/uploads')).toBe(false);
});
it('returns false for non-string / empty inputs', () => {
expect(isPathInsideDir('/data/uploads', '')).toBe(false);
expect(isPathInsideDir('', '/data/uploads/foo.png')).toBe(false);
expect(isPathInsideDir(null, undefined)).toBe(false);
});
});
describe('assertSafeFilename', () => {
it('accepts a safe basename with an allowlisted extension', () => {
expect(() => assertSafeFilename('foo.png', { extensions: ['.png'] })).not.toThrow();
expect(() => assertSafeFilename('lora-cool.safetensors', { extensions: ['.safetensors'] })).not.toThrow();
});
it('matches extensions case-insensitively', () => {
expect(() => assertSafeFilename('FOO.PNG', { extensions: ['.png'] })).not.toThrow();
expect(() => assertSafeFilename('cool.SafeTensors', { extensions: ['.safetensors'] })).not.toThrow();
});
it('allows substring `..` in the middle of a name', () => {
expect(() => assertSafeFilename('my..render.png', { extensions: ['.png'] })).not.toThrow();
});
it('rejects path separators', () => {
expect(() => assertSafeFilename('sub/foo.png', { extensions: ['.png'] })).toThrow(/Invalid filename/);
expect(() => assertSafeFilename('sub\\foo.png', { extensions: ['.png'] })).toThrow(/Invalid filename/);
});
it('rejects exact-traversal `.` and `..`', () => {
expect(() => assertSafeFilename('.', { extensions: ['.png'] })).toThrow(/Invalid filename/);
expect(() => assertSafeFilename('..', { extensions: ['.png'] })).toThrow(/Invalid filename/);
});
it('rejects null bytes', () => {
expect(() => assertSafeFilename('foo\0.png', { extensions: ['.png'] })).toThrow(/Invalid filename/);
});
it('rejects empty or non-string inputs', () => {
expect(() => assertSafeFilename('', { extensions: ['.png'] })).toThrow(/Filename required/);
expect(() => assertSafeFilename(undefined, { extensions: ['.png'] })).toThrow(/Filename required/);
expect(() => assertSafeFilename(null, { extensions: ['.png'] })).toThrow(/Filename required/);
});
it('rejects unrecognized extensions', () => {
expect(() => assertSafeFilename('foo.jpg', { extensions: ['.png'] })).toThrow(/Invalid filename/);
expect(() => assertSafeFilename('foo.exe', { extensions: ['.png', '.gif'] })).toThrow(/Invalid filename/);
});
it('uses subject in error messages', () => {
expect(() => assertSafeFilename('', { extensions: ['.safetensors'], subject: 'LoRA filename' }))
.toThrow(/LoRA filename required/);
expect(() => assertSafeFilename('foo.jpg', { extensions: ['.safetensors'], subject: 'LoRA filename' }))
.toThrow(/Invalid LoRA filename/);
});
it('throws on missing extensions option (programmer error, not user)', () => {
expect(() => assertSafeFilename('foo.png', {})).toThrow(/extensions allowlist is required/);
expect(() => assertSafeFilename('foo.png', { extensions: [] })).toThrow(/extensions allowlist is required/);
});
it('throws on extensions that do not start with a dot (programmer error)', () => {
// Bare suffix like 'png' would also match 'not-an-imagepng' if we didn't
// enforce the leading-dot rule — that's a serious validation hole.
expect(() => assertSafeFilename('foo.png', { extensions: ['png'] }))
.toThrow(/each extension must be a non-empty string starting with/);
expect(() => assertSafeFilename('foo.png', { extensions: ['.png', 'jpg'] }))
.toThrow(/each extension must be a non-empty string starting with/);
expect(() => assertSafeFilename('foo.png', { extensions: [''] }))
.toThrow(/each extension must be a non-empty string starting with/);
expect(() => assertSafeFilename('foo.png', { extensions: ['.'] }))
.toThrow(/each extension must be a non-empty string starting with/);
expect(() => assertSafeFilename('foo.png', { extensions: [123] }))
.toThrow(/each extension must be a non-empty string starting with/);
});
// `isSafeFilename` is the same rule as a predicate, for read-only callers
// that must report a bad name as data. The two share one implementation, so
// the contract worth pinning is that they never disagree — a future edit to
// either must keep the gate and the reported verdict in lockstep.
it('agrees with isSafeFilename on every input class', () => {
const extensions = ['.png', '.mp3'];
const cases = [
'foo.png', 'FOO.PNG', 'x.mp3', 'my..render.png', // accepted
'foo.jpg', 'png', 'foo.png.txt', // bad extension
'sub/foo.png', 'sub\\foo.png', '/foo.png', '../foo.png', '..\\foo.png', // separators
'.', '..', // exact traversal
'foo\0.png', '\0', // null bytes
'', null, undefined, 0, false, [], {}, // missing / non-string
];
for (const name of cases) {
let threw = false;
try {
assertSafeFilename(name, { extensions });
} catch {
threw = true;
}
expect(isSafeFilename(name, extensions), `disagreed on ${JSON.stringify(name)}`)
.toBe(!threw);
}
});
it('honors requiredMessage override for the missing-input case only', () => {
// Backward-compat path: wrappers that used to throw a fixed phrase
// (e.g. "Filename required" / "Invalid filename") can preserve that
// message without affecting the invalid-input message.
expect(() => assertSafeFilename('', {
extensions: ['.safetensors'],
subject: 'LoRA filename',
requiredMessage: 'Filename required',
})).toThrow(/^Filename required$/);
// Invalid path still uses the subject-derived message.
expect(() => assertSafeFilename('foo.jpg', {
extensions: ['.safetensors'],
subject: 'LoRA filename',
requiredMessage: 'Filename required',
})).toThrow(/^Invalid LoRA filename$/);
});
it('attaches 400 status + VALIDATION_ERROR code on the thrown ServerError', () => {
try {
assertSafeFilename('bad/path.png', { extensions: ['.png'] });
throw new Error('Expected assertion to throw');
} catch (err) {
expect(err.status).toBe(400);
expect(err.code).toBe('VALIDATION_ERROR');
}
});
});
describe('listDirectoryByExtension', () => {
const tmpRoot = join(tmpdir(), `portos-listdir-test-${process.pid}-${Date.now()}`);
beforeEach(async () => {
await rm(tmpRoot, { recursive: true, force: true }).catch(() => {});
await mkdir(tmpRoot, { recursive: true });
});
afterEach(async () => {
await rm(tmpRoot, { recursive: true, force: true }).catch(() => {});
});
it('returns [] when the directory does not exist', async () => {
const res = await listDirectoryByExtension(join(tmpRoot, 'missing'), {
extensions: ['.png'],
mapEntry: (n) => ({ filename: n }),
});
expect(res).toEqual([]);
});
it('filters by extension (case-insensitive) and maps survivors', async () => {
await writeFile(join(tmpRoot, 'a.png'), 'a');
await writeFile(join(tmpRoot, 'b.PNG'), 'b');
await writeFile(join(tmpRoot, 'c.jpg'), 'c');
const res = await listDirectoryByExtension(tmpRoot, {
extensions: ['.png'],
mapEntry: (name, _full, s) => ({ name, sizeBytes: s.size }),
});
const names = res.map((r) => r.name).sort();
expect(names).toEqual(['a.png', 'b.PNG']);
});
it('drops directories when requireRegularFile is true (default)', async () => {
await writeFile(join(tmpRoot, 'real.safetensors'), 'data');
await mkdir(join(tmpRoot, 'fake.safetensors'));
const res = await listDirectoryByExtension(tmpRoot, {
extensions: ['.safetensors'],
mapEntry: (name) => ({ name }),
});
expect(res).toEqual([{ name: 'real.safetensors' }]);
});
it('keeps directories when requireRegularFile is false (gallery legacy)', async () => {
await writeFile(join(tmpRoot, 'real.png'), 'data');
await mkdir(join(tmpRoot, 'fake.png'));
const res = await listDirectoryByExtension(tmpRoot, {
extensions: ['.png'],
requireRegularFile: false,
mapEntry: (name) => ({ name }),
});
const names = res.map((r) => r.name).sort();
expect(names).toEqual(['fake.png', 'real.png']);
});
it('drops entries whose mapEntry returns null', async () => {
await writeFile(join(tmpRoot, 'a.json'), 'a');
await writeFile(join(tmpRoot, 'b.json'), 'b');
const res = await listDirectoryByExtension(tmpRoot, {
extensions: ['.json'],
mapEntry: (name) => (name === 'a.json' ? null : { name }),
});
expect(res).toEqual([{ name: 'b.json' }]);
});
it('throws if extensions is missing or empty', async () => {
await expect(
listDirectoryByExtension(tmpRoot, { mapEntry: (n) => n }),
).rejects.toThrow(/extensions allowlist/);
await expect(
listDirectoryByExtension(tmpRoot, { extensions: [], mapEntry: (n) => n }),
).rejects.toThrow(/extensions allowlist/);
});
it('throws if mapEntry is not a function', async () => {
await expect(
listDirectoryByExtension(tmpRoot, { extensions: ['.png'] }),
).rejects.toThrow(/mapEntry must be a function/);
});
});
describe('dirSize strict scans', () => {
afterEach(() => restoreFsMocks());
it('rejects an unreadable path instead of reporting a real zero', async () => {
const error = Object.assign(new Error('permission denied'), { code: 'EACCES' });
fsPromises.stat.mockRejectedValue(error);
await expect(dirSize('/example/unreadable', { strict: true })).rejects.toBe(error);
});
it('treats a confirmed missing path as an empty directory', async () => {
fsPromises.stat.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' }));
await expect(dirSize('/example/missing', { strict: true })).resolves.toBe(0);
});
});
// CONVENTION: this block writes fixtures into the REAL `data/images`,
// `data/image-refs`, and `data/templates` roots because `PATHS` is module-
// evaluated and not easily overridable. Two rules to keep dev/CI worktrees
// clean and avoid clobbering shipped assets:
// 1. Every fixture name MUST start with the `fileutils-test-` prefix so
// the cleanup below can target it unambiguously (and so the basenames
// don't shadow any real asset the resolver would otherwise find).
// 2. Cleanup goes in `afterAll`, NOT a recursive remove of the root —
// the data/ roots hold the user's universe content and shipped templates.
describe('resolveImageInputPath', () => {
const sampleTemplate = join(__dirname_test, '..', '..', 'data.reference', 'templates', 'character-reference-sheet.png');
const galleryName = 'fileutils-test-gallery.png';
const refsName = 'fileutils-test-refs.png';
const templateName = 'fileutils-test-template.png';
const galleryPath = join(PATHS.images, galleryName);
const refsPath = join(PATHS.imageRefs, refsName);
const templatePath = join(PATHS.visualTemplates, templateName);
beforeEach(() => {
// Provision fixtures in each approved root so the resolver can find
// them. Reuses the shipped sample asset as a stand-in PNG body.
for (const root of [PATHS.images, PATHS.imageRefs, PATHS.visualTemplates]) {
if (!existsSync(root)) mkdirSync(root, { recursive: true });
}
if (existsSync(sampleTemplate)) {
if (!existsSync(galleryPath)) copyFileSync(sampleTemplate, galleryPath);
if (!existsSync(refsPath)) copyFileSync(sampleTemplate, refsPath);
if (!existsSync(templatePath)) copyFileSync(sampleTemplate, templatePath);
}
});
afterAll(() => {
// Remove ONLY the per-test fixture files (uniquely-named so dev/CI
// worktrees aren't polluted and later basename lookups don't keep
// finding stale resolver hits). Never recursively remove the real
// `data/images` / `data/image-refs` / `data/templates` roots.
for (const p of [galleryPath, refsPath, templatePath]) {
if (existsSync(p)) rmSync(p, { force: true });
}
});
it('returns null for non-string / empty input', () => {
expect(resolveImageInputPath(null)).toBeNull();
expect(resolveImageInputPath('')).toBeNull();
expect(resolveImageInputPath(undefined)).toBeNull();
expect(resolveImageInputPath(123)).toBeNull();
});
it('resolves a basename present in the gallery (first root)', () => {
const out = resolveImageInputPath(galleryName);
expect(out).toBeTruthy();
expect(out).toContain(join('data', 'images'));
expect(out).toContain(galleryName);
});
it('resolves a basename present only in image-refs', () => {
const out = resolveImageInputPath(refsName);
expect(out).toBeTruthy();
expect(out).toContain(join('data', 'image-refs'));
});
it('resolves a basename present only in visualTemplates', () => {
const out = resolveImageInputPath(templateName);
// `templateName` is unique to the visualTemplates root (it is NOT copied
// into gallery or image-refs), so the resolver must land on the third
// root and the returned path must carry that root's segment + basename —
// a plain truthiness check would pass even on a wrong-root resolution.
expect(out).toContain(join('data', 'templates'));
expect(out).toContain(templateName);
});
it('REGRESSION: absolute path under a specific root stays in that root', () => {
// Bug it guards: previously the resolver basenamed any input and tried
// each root in order — so `/data/templates/<name>.png` for a file that
// also exists in `/data/images/<name>.png` would silently redirect to
// the gallery copy. Reference-sheet renders would have used the wrong
// init image. Verify each absolute path resolves to its own root.
const galleryAbs = join(PATHS.images, galleryName);
const refsAbs = join(PATHS.imageRefs, refsName);
const templateAbs = join(PATHS.visualTemplates, templateName);
expect(resolveImageInputPath(galleryAbs)).toContain(join('data', 'images'));
expect(resolveImageInputPath(refsAbs)).toContain(join('data', 'image-refs'));
expect(resolveImageInputPath(templateAbs)).toContain(join('data', 'templates'));
});
it('REGRESSION: same basename in multiple roots — absolute path picks the matching root', () => {
// All three fixtures share the same body (copied from sampleTemplate),
// but the absolute path should pin to its own root, NOT collapse to
// the gallery via basename fallback.
const refsAbs = join(PATHS.imageRefs, refsName);
const out = resolveImageInputPath(refsAbs);
expect(out).toContain(join('data', 'image-refs'));
expect(out).not.toContain(join('data', 'images'));
});
});
describe('resolveScreenshot', () => {
const sampleTemplate = join(__dirname_test, '..', '..', 'data.reference', 'templates', 'character-reference-sheet.png');
const shotName = 'fileutils-test-screenshot.png';
const shotPath = join(PATHS.screenshots, shotName);
beforeEach(() => {
if (!existsSync(PATHS.screenshots)) mkdirSync(PATHS.screenshots, { recursive: true });
if (existsSync(sampleTemplate) && !existsSync(shotPath)) copyFileSync(sampleTemplate, shotPath);
});
afterAll(() => {
// Remove ONLY the uniquely-named fixture — never the real screenshots root.
if (existsSync(shotPath)) rmSync(shotPath, { force: true });
});
it('resolves a basename present under the screenshots root', () => {
const out = resolveScreenshot(shotName);
expect(out).toBeTruthy();
expect(out).toContain(join('data', 'screenshots'));
expect(out).toContain(shotName);
});
it('SECURITY: rejects parent-directory traversal that escapes the screenshots root', () => {
// `loadImageAsBase64` (issue #1820) fed `imagePath` straight from req.body;
// a `../` payload must NOT resolve to a file outside data/screenshots.
expect(resolveScreenshot('../../etc/passwd')).toBeNull();
expect(resolveScreenshot('../package.json')).toBeNull();
});
it('SECURITY: rejects an absolute path outside the screenshots root', () => {
expect(resolveScreenshot('/etc/passwd')).toBeNull();
expect(resolveScreenshot('/etc/hosts')).toBeNull();
});
it('rejects non-image extensions and missing files', () => {
expect(resolveScreenshot('notes.txt')).toBeNull();
expect(resolveScreenshot('does-not-exist.png')).toBeNull();
});
it('returns null for non-string / empty input', () => {
expect(resolveScreenshot(null)).toBeNull();
expect(resolveScreenshot('')).toBeNull();
expect(resolveScreenshot(undefined)).toBeNull();
});
});
describe('sha256File', () => {
let dir;
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'portos-sha256-')); });
afterEach(() => { rmSync(dir, { recursive: true, force: true }); });
it('hashes a small file in one shot and matches createHash digest', async () => {
const p = join(dir, 'small.bin');
writeFileSync(p, 'hello world');
const expected = createHash('sha256').update('hello world').digest('hex');
expect(await sha256File(p)).toBe(expected);
});
it('hashes a large file via streaming (>= 512KB)', async () => {
const p = join(dir, 'big.bin');
const buf = Buffer.alloc(600 * 1024, 0x42);
writeFileSync(p, buf);
const expected = createHash('sha256').update(buf).digest('hex');
expect(await sha256File(p)).toBe(expected);
});
it('returns identical digests for identical content under different paths', async () => {
const a = join(dir, 'a.bin'); const b = join(dir, 'b.bin');
writeFileSync(a, 'same-content');
writeFileSync(b, 'same-content');
expect(await sha256File(a)).toBe(await sha256File(b));
});
it('returns different digests for different content', async () => {
const a = join(dir, 'a.bin'); const b = join(dir, 'b.bin');
writeFileSync(a, 'one');
writeFileSync(b, 'two');
expect(await sha256File(a)).not.toBe(await sha256File(b));
});
});
describe('readFileTail', () => {
let dir;
beforeEach(() => { dir = mkdtempSync(join(tmpdir(), 'portos-tail-')); });
afterEach(() => { rmSync(dir, { recursive: true, force: true }); });
it('returns null for a missing file so it stays distinct from an empty one', async () => {
expect(await readFileTail(join(dir, 'nope.txt'), 1024)).toBeNull();
});
it('returns an empty string for a zero-byte file', async () => {
const p = join(dir, 'empty.txt');
writeFileSync(p, '');
expect(await readFileTail(p, 1024)).toBe('');
});
it('returns the whole file when it fits under maxBytes', async () => {
const p = join(dir, 'small.txt');
writeFileSync(p, 'hello');
expect(await readFileTail(p, 1024)).toBe('hello');
});
it('returns only the trailing window when the file is larger than maxBytes', async () => {
const p = join(dir, 'big.txt');
writeFileSync(p, 'abcdefghij');
expect(await readFileTail(p, 4)).toBe('ghij');
});
it('does not load the whole file for a multi-MB log', async () => {
const p = join(dir, 'huge.log');
writeFileSync(p, Buffer.alloc(4 * 1024 * 1024, 0x61));
const tail = await readFileTail(p, 128);
expect(tail).toHaveLength(128);
});
});
describe('expandHome', () => {
it('expands a bare `~` to the homedir', () => {
expect(expandHome('~')).toBe(homedir());
});
it('expands `~/foo` to homedir + foo', () => {
const out = expandHome('~/foo');
expect(out.startsWith(homedir())).toBe(true);
expect(out.endsWith('foo')).toBe(true);
});
it('expands the Windows form `~\\foo` so `lib/fileUtils.js` stays cross-platform', () => {
const out = expandHome('~\\foo');
expect(out.startsWith(homedir())).toBe(true);
expect(out.endsWith('foo')).toBe(true);
});
it('preserves absolute, relative, and empty inputs', () => {
expect(expandHome('/abs/path')).toBe('/abs/path');
expect(expandHome('relative/path')).toBe('relative/path');
expect(expandHome('')).toBe('');
});
it('preserves non-string inputs (null / undefined / number) without throwing', () => {
expect(expandHome(null)).toBe(null);
expect(expandHome(undefined)).toBe(undefined);
expect(expandHome(42)).toBe(42);
});
it('only expands a leading `~` — embedded `~` chars are preserved (iCloud~md~obsidian)', () => {
expect(expandHome('iCloud~md~obsidian')).toBe('iCloud~md~obsidian');
expect(expandHome('foo/~bar')).toBe('foo/~bar');
});
});
describe('ensureDir', () => {
let tmpRoot;
beforeEach(() => {
tmpRoot = mkdtempSync(join(tmpdir(), 'fileutils-ensuredir-'));
});
afterEach(() => {
fsPromises.mkdir.mockClear();
rmSync(tmpRoot, { recursive: true, force: true });
});
it('creates a missing nested directory', async () => {
const target = join(tmpRoot, 'a', 'b', 'c');
await ensureDir(target);
expect(existsSync(target)).toBe(true);
});
it('is idempotent — succeeds when the directory already exists', async () => {
const target = join(tmpRoot, 'exists');
await ensureDir(target);
await expect(ensureDir(target)).resolves.toBeUndefined();
});
it('swallows a spurious Windows mkdir error when the dir already exists (regression)', async () => {
const target = join(tmpRoot, 'preexisting');
mkdirSync(target);
// Simulate the Windows UNKNOWN/EPERM that fs.mkdir can throw even on an
// existing directory (antivirus locks, OneDrive sync, mapped drives).
const spurious = Object.assign(new Error('UNKNOWN: unknown error, mkdir'), { code: 'UNKNOWN' });
fsPromises.mkdir.mockRejectedValueOnce(spurious);
await expect(ensureDir(target)).resolves.toBeUndefined();
});
it('still rejects when mkdir fails and the path is not a directory afterward', async () => {
const target = join(tmpRoot, 'never-created');
const realFailure = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' });
fsPromises.mkdir.mockRejectedValueOnce(realFailure);
await expect(ensureDir(target)).rejects.toThrow(/EACCES/);
});
});
describe('atomicWrite', () => {
let tmpRoot;
beforeEach(() => {
tmpRoot = mkdtempSync(join(tmpdir(), 'fileutils-atomicwrite-'));
});
afterEach(() => {
restoreFsMocks();
rmSync(tmpRoot, { recursive: true, force: true });
});
it('serializes a plain object with 2-space indentation', async () => {
const target = join(tmpRoot, 'obj.json');
await atomicWrite(target, { b: 2, a: 1 });
expect(await readFile(target, 'utf8')).toBe('{\n "b": 2,\n "a": 1\n}');
});
it('writes a string payload verbatim (preserving a trailing newline)', async () => {
const target = join(tmpRoot, 'raw.json');
await atomicWrite(target, '{"x":1}\n');
expect(await readFile(target, 'utf8')).toBe('{"x":1}\n');
});
// Crash durability (#12025): the bytes reach the disk before the rename
// publishes them, and the directory entry reaches the disk after. The
// ordering is observed through the fs seam because a real power loss
// cannot be staged in a unit test.
it('fsyncs the temp file before the rename and the parent directory after it', async () => {
const target = join(tmpRoot, 'durable.json');
const calls = [];
fsPromises.writeFile.mockImplementationOnce(async (path, data, options) => {
calls.push(['writeFile', options?.flush]);
return realFsPromises.writeFile(path, data, options);
});
fsPromises.rename.mockImplementationOnce(async (...args) => {
calls.push(['rename']);
return realFsPromises.rename(...args);
});
fsPromises.open.mockImplementationOnce(async (path, flags) => {
const handle = await realFsPromises.open(path, flags);
const sync = handle.sync.bind(handle);
handle.sync = async () => { calls.push(['dirSync', path]); return sync(); };
return handle;
});
await atomicWrite(target, { durable: true });
// Windows cannot open a directory for fsync, so syncDirectory skips it there.
const expectedCalls = [['writeFile', true], ['rename'], ...(process.platform === 'win32' ? [] : [['dirSync', tmpRoot]])];
expect(calls).toEqual(expectedCalls);
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ durable: true });
});
it.skipIf(process.platform === 'win32')('keeps a published write when the directory cannot be fsynced, warning once per directory', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
try {
const target = join(tmpRoot, 'netfs.json');
const failure = Object.assign(new Error('synthetic directory fsync failure'), { code: 'EINVAL' });
fsPromises.open.mockRejectedValueOnce(failure).mockRejectedValueOnce(failure);
await atomicWrite(target, { v: 1 });
await atomicWrite(target, { v: 2 });
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ v: 2 });
expect(warn).toHaveBeenCalledTimes(1);
expect(warn.mock.calls[0][0]).toContain('EINVAL');
} finally {
warn.mockRestore();
}
});
it('removes a partially written temp file after ENOSPC and allows a later retry', async () => {
const target = join(tmpRoot, 'state.json');
writeFileSync(target, 'original');
const failure = Object.assign(new Error('synthetic disk full'), { code: 'ENOSPC' });
fsPromises.writeFile.mockImplementationOnce(async (path, _data, options) => {
await realFsPromises.writeFile(path, 'partial', options);
throw failure;
});
await expect(atomicWrite(target, 'replacement')).rejects.toBe(failure);
expect(readFileSync(target, 'utf8')).toBe('original');
expect(readdirSync(tmpRoot)).toEqual(['state.json']);
await atomicWrite(target, 'replacement');
expect(readFileSync(target, 'utf8')).toBe('replacement');
expect(readdirSync(tmpRoot)).toEqual(['state.json']);
});
it('preserves the destination when its permissions cannot be read', async () => {
const target = join(tmpRoot, 'private.json');
writeFileSync(target, 'original', { mode: 0o600 });
const originalMode = (await realFsPromises.stat(target)).mode;
const failure = Object.assign(new Error('synthetic stat failure'), { code: 'EIO' });
fsPromises.stat.mockRejectedValueOnce(failure);
await expect(atomicWrite(target, 'replacement')).rejects.toBe(failure);
expect(readFileSync(target, 'utf8')).toBe('original');
expect((await realFsPromises.stat(target)).mode).toBe(originalMode);
expect(readdirSync(tmpRoot)).toEqual(['private.json']);
});
it('preserves the destination and removes the temp file when restoring its mode fails', async () => {
const target = join(tmpRoot, 'private.json');
writeFileSync(target, 'original', { mode: 0o600 });
const failure = Object.assign(new Error('synthetic chmod failure'), { code: 'EACCES' });
fsPromises.chmod.mockRejectedValueOnce(failure);
await expect(atomicWrite(target, 'replacement')).rejects.toBe(failure);
expect(readFileSync(target, 'utf8')).toBe('original');
expect(readdirSync(tmpRoot)).toEqual(['private.json']);
});
it.skipIf(process.platform === 'win32')(
'preserves the destination file\'s restrictive mode on rewrite (regression: #1837)',
async () => {
const target = join(tmpRoot, 'secret.json');
// Seed a hand-restricted secret file (e.g. an OAuth tokens.json the user
// chmod 600'd). The temp-write + rename must NOT widen it to the umask
// default — a plain writeFile(existing) kept the inode's mode, and
// atomicWrite mirrors that.
writeFileSync(target, '{"token":"old"}', { mode: 0o600 });
const { chmodSync } = await import('fs');
chmodSync(target, 0o600); // ensure 600 regardless of umask at creation
await atomicWrite(target, { token: 'new' });
const { statSync } = await import('fs');
expect(statSync(target).mode & 0o777).toBe(0o600);
expect(JSON.parse(await readFile(target, 'utf8'))).toEqual({ token: 'new' });
}
);
it.skipIf(process.platform === 'win32')(
'creates the temp file with the destination\'s mode so a secret is never briefly world-readable',
async () => {
const target = join(tmpRoot, 'secret.json');
writeFileSync(target, '{"token":"old"}', { mode: 0o600 });
await realFsPromises.chmod(target, 0o600); // independent of the creation umask
let modeAtCreation = null;
let tempPath = null;
// The final chmod would mask a missing creation mode, so observe the
// temp file straight after its first write, before chmod/rename run.
fsPromises.writeFile.mockImplementationOnce(async (path, data, options) => {
await realFsPromises.writeFile(path, data, options);
tempPath = path;
modeAtCreation = (await realFsPromises.stat(path)).mode & 0o777;
});
await atomicWrite(target, { token: 'new' });
expect(tempPath).not.toBe(target);
expect(modeAtCreation).toBe(0o600);
expect((await realFsPromises.stat(target)).mode & 0o777).toBe(0o600);
}
);
it('creates a new file at the default mode when the destination does not exist', async () => {
const target = join(tmpRoot, 'fresh.json');
await atomicWrite(target, { created: true });
expect(JSON.parse(await readFile(target, 'utf8'))).toEqual({ created: true });
});
it.skipIf(process.platform === 'win32')(
'replaces a symlink target with a regular file rather than following it (design decision: #1893)',
async () => {
// Decision pinned by this test: temp+rename REPLACES the link, standard
// atomic-write semantics — it does NOT follow the symlink to update the
// backing file. Following would reintroduce the non-atomic in-place
// truncate atomicWrite exists to avoid. See the JSDoc + issue #1893.
const { symlinkSync, lstatSync } = await import('fs');
const backing = join(tmpRoot, 'backing.json');
const link = join(tmpRoot, 'link.json');
writeFileSync(backing, '{"orig":true}');
symlinkSync(backing, link);
await atomicWrite(link, { replaced: true });
// The link path is now a regular file holding the new content...
expect(lstatSync(link).isSymbolicLink()).toBe(false);
expect(JSON.parse(await readFile(link, 'utf8'))).toEqual({ replaced: true });
// ...and the original backing file is untouched (link was not followed).
expect(JSON.parse(await readFile(backing, 'utf8'))).toEqual({ orig: true });
}
);
});
});
// =============================================================================
// Windows swap-window retries (#4095)
//
// On win32 `atomicWrite`'s fallback backup swap renames the destination away and
// back, so for an instant the file DOES NOT EXIST. A read landing in that window
// used to get ENOENT and report it as a trustworthy "nothing here yet", handing
// the caller its default instead of the real contents. These tests pin both
// halves of the fix and that POSIX still pays nothing for it.
// =============================================================================
describe('Windows swap-window retries (#4095)', () => {
// Matches WIN_RETRY_ATTEMPTS in fileCore.js.
const RETRY_ATTEMPTS = 5;
const lockError = (code) => Object.assign(new Error(`${code}: simulated windows lock`), { code });
let tmpRoot;
let restorePlatform = null;
// Parks the restore for afterEach; each case pins exactly once.
const fakePlatform = (value) => { restorePlatform = pinPlatform(value); };
beforeEach(() => {
tmpRoot = mkdtempSync(join(tmpdir(), 'fileutils-winswap-'));
// Earlier describes in this file exercise the same fs mocks — start from a
// clean call log so the call-count assertions below mean what they say.
fsPromises.readFile.mockClear();
fsPromises.rename.mockClear();
fsPromises.readdir.mockClear();
});
afterEach(() => {
if (restorePlatform) restorePlatform();
restorePlatform = null;
// Drop any unconsumed `*Once` queues AND re-establish the delegating
// implementations the rest of this file relies on.
restoreFsMocks();
rmSync(tmpRoot, { recursive: true, force: true });
});
describe('atomicWrite', () => {
it('fsyncs the temp file but skips the directory sync, which Windows cannot open a handle for (#12025)', async () => {
const target = join(tmpRoot, 'state.json');
fakePlatform('win32');
fsPromises.open.mockClear();
await atomicWrite(target, { v: 1 });
expect(fsPromises.writeFile).toHaveBeenLastCalledWith(expect.any(String), expect.any(String), expect.objectContaining({ flush: true }));
expect(fsPromises.open).not.toHaveBeenCalled();
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ v: 1 });
});
it('retries the atomic rename on a transient lock instead of opening the destination-missing window', async () => {
const target = join(tmpRoot, 'state.json');
writeFileSync(target, JSON.stringify({ v: 1 }));
fakePlatform('win32');
fsPromises.rename.mockRejectedValueOnce(lockError('EPERM'));
await atomicWrite(target, { v: 2 });
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ v: 2 });
// The backup swap is identified by a rename whose SOURCE is the
// destination itself (`rename(filePath, bak)`). The retry path must never
// move the destination aside — that is the window this fix closes.
const movedDestinationAside = fsPromises.rename.mock.calls.some(([from]) => from === target);
expect(movedDestinationAside).toBe(false);
expect(fsPromises.rename).toHaveBeenCalledTimes(2);
});
it('retries a transient EBUSY lock on the atomic rename', async () => {
const target = join(tmpRoot, 'busy.json');
writeFileSync(target, JSON.stringify({ v: 1 }));
fakePlatform('win32');
fsPromises.rename.mockRejectedValueOnce(lockError('EBUSY'));
await atomicWrite(target, { v: 2 });
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ v: 2 });
expect(fsPromises.rename).toHaveBeenCalledTimes(2);
expect(fsPromises.rename.mock.calls.some(([from]) => from === target)).toBe(false);
});
it('preserves the original bytes and removes only the temp file when every retry stays locked (#9556)', async () => {
const target = join(tmpRoot, 'stubborn.json');
writeFileSync(target, JSON.stringify({ v: 1 }));
fakePlatform('win32');
fsPromises.rename.mockRejectedValue(lockError('EPERM'));
await expect(atomicWrite(target, { v: 2 })).rejects.toMatchObject({
code: 'EPERM',
message: expect.stringContaining('existing file was left untouched'),
});
// The destination is never moved aside — no backup swap exists any more.
expect(fsPromises.rename.mock.calls.some(([from]) => from === target)).toBe(false);
expect(fsPromises.rename).toHaveBeenCalledTimes(RETRY_ATTEMPTS);
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ v: 1 });
expect(readdirSync(tmpRoot)).toEqual(['stubborn.json']);
});
it('does not retry off win32 — a rename failure still surfaces immediately', async () => {
const target = join(tmpRoot, 'posix.json');
writeFileSync(target, JSON.stringify({ v: 1 }));
// Pinned explicitly, not inherited from the host — this suite also runs on
// a real Windows CI runner, where the un-faked platform IS win32.
fakePlatform('linux');
fsPromises.rename.mockRejectedValueOnce(lockError('EPERM'));
await expect(atomicWrite(target, { v: 2 })).rejects.toThrow(/EPERM/);
expect(fsPromises.rename).toHaveBeenCalledTimes(1);
// The original file is untouched and no temp debris is left behind.
expect(JSON.parse(readFileSync(target, 'utf8'))).toEqual({ v: 1 });
expect(readdirSync(tmpRoot)).toEqual(['posix.json']);
});
});
describe('readJSONFileStrict', () => {
it('retries a locked read on win32 and returns the real contents as trustworthy', async () => {
const target = join(tmpRoot, 'locked.json');
writeFileSync(target, JSON.stringify({ real: true }));
fakePlatform('win32');
fsPromises.readFile.mockRejectedValueOnce(lockError('EBUSY'));
expect(await readJSONFileStrict(target, { fallback: true })).toEqual({ ok: true, value: { real: true } });
expect(fsPromises.readFile).toHaveBeenCalledTimes(2);
});
it('retries an ENOENT on win32 when a swap sibling shows a write is in flight', async () => {
const target = join(tmpRoot, 'swapping.json');
writeFileSync(target, JSON.stringify({ real: true }));
// The debris `atomicWrite` leaves visible while a swap is mid-flight.
writeFileSync(`${target}.1234.5678.bak`, '{"real":true}');
fakePlatform('win32');
fsPromises.readFile.mockRejectedValueOnce(lockError('ENOENT'));
expect(await readJSONFileStrict(target, { fallback: true })).toEqual({ ok: true, value: { real: true } });
expect(fsPromises.readdir).toHaveBeenCalled();
});
it('matches a swap sibling whose casing differs — NTFS/FAT are case-insensitive', async () => {
const target = join(tmpRoot, 'casing.json');
writeFileSync(target, JSON.stringify({ real: true }));
// The writer may have created the file (and therefore its swap debris)
// under different casing than the reader's path — legal on Windows.
writeFileSync(join(tmpRoot, 'Casing.json.1234.5678.BAK'), '{"real":true}');
fakePlatform('win32');
fsPromises.readFile.mockRejectedValueOnce(lockError('ENOENT'));
expect(await readJSONFileStrict(target, { fallback: true })).toEqual({ ok: true, value: { real: true } });
});
it('does NOT treat a .tmp sibling as a swap — a first-ever write is not a vanished file', async () => {
// `atomicWrite` writes a `.tmp` on EVERY write, including the first create
// of a file that has no destination yet. Retrying on that would make a
// legitimate "not written yet" read block for the in-flight write and
// return its brand-new contents. Only the `.bak` marks the swap window.
const missing = join(tmpRoot, 'first-write.json');
writeFileSync(`${missing}.4321.8765.tmp`, '{"brand":"new"}');
fakePlatform('win32');
expect(await readJSONFileStrict(missing, { fallback: true })).toEqual({ ok: true, value: { fallback: true } });
expect(fsPromises.readFile).toHaveBeenCalledTimes(1);
});
it('reports a missing file with a legacy .bak sibling as recovery-required, never empty (#9556)', async () => {
const target = join(tmpRoot, 'orphaned.json');
writeFileSync(`${target}.1234.5678.bak`, '{"secret":"original"}');
fakePlatform('win32');
expect(await readJSONFileStrict(target, { fallback: true }, { logError: false }))
.toEqual({ ok: false, value: { fallback: true } });
expect(await tryReadFileStrict(target)).toEqual({ ok: false, value: null });
});
it('does not retry a plain missing file on win32 — no sibling, no second read', async () => {
const missing = join(tmpRoot, 'never-written.json');
fakePlatform('win32');
expect(await readJSONFileStrict(missing, { fallback: true })).toEqual({ ok: true, value: { fallback: true } });
expect(fsPromises.readFile).toHaveBeenCalledTimes(1);
});
it('costs a plain missing file nothing off win32 — one read, no directory scan', async () => {
const missing = join(tmpRoot, 'never-written.json');
fakePlatform('linux');
expect(await readJSONFileStrict(missing, { fallback: true })).toEqual({ ok: true, value: { fallback: true } });
expect(fsPromises.readFile).toHaveBeenCalledTimes(1);
expect(fsPromises.readdir).not.toHaveBeenCalled();
});
it('does not retry a locked read off win32 — the read stays untrustworthy', async () => {
const target = join(tmpRoot, 'posix-locked.json');
writeFileSync(target, JSON.stringify({ real: true }));
fakePlatform('linux');
fsPromises.readFile.mockRejectedValueOnce(lockError('EBUSY'));
expect(await readJSONFileStrict(target, { fallback: true }, { logError: false }))
.toEqual({ ok: false, value: { fallback: true } });
expect(fsPromises.readFile).toHaveBeenCalledTimes(1);
});
});
});
// =============================================================================
// createCachedStore — cached JSON store with serialized writes (#2539)
// =============================================================================
describe('createCachedStore', () => {
let dir;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'cached-store-test-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
it('loads the default when the file is absent, then round-trips a save', async () => {
const store = createCachedStore(join(dir, 'x.json'), { n: 0 });
expect(await store.load()).toEqual({ n: 0 });
await store.save({ n: 5 });
expect(await store.load()).toEqual({ n: 5 });
});
it('mutate persists the fn result and returns it', async () => {
const store = createCachedStore(join(dir, 'x.json'), { n: 0 });
const out = await store.mutate((data) => { data.n = 9; });
expect(out).toEqual({ n: 9 });
// Persisted to disk, not just cache
expect(JSON.parse(await readFile(join(dir, 'x.json'), 'utf-8'))).toEqual({ n: 9 });
});
it('serializes concurrent read-modify-write so no update is lost', async () => {
const file = join(dir, 'counter.json');
const store = createCachedStore(file, { n: 0 }, { ttl: 0 });
// ttl:0 forces every load to re-read from disk — the worst case for the
// classic load→mutate→save clobber. Without serialization, N concurrent
// increments race down to ~1; mutate() must land all N.
const N = 25;
await Promise.all(
Array.from({ length: N }, () => store.mutate(async (data) => {
const current = data.n;
await Promise.resolve(); // yield — invites interleaving
data.n = current + 1;
}))
);
expect((await store.load()).n).toBe(N);
expect(JSON.parse(await readFile(file, 'utf-8')).n).toBe(N);
});
it('does not let an in-flight stale read replace a persisted mutation', async () => {
const file = join(dir, 'feed-state.json');
const store = createCachedStore(file, { items: [] }, { ttl: 60_000 });
await store.save({ items: [] });
store.invalidateCache();
let releaseRead;
let signalReadStarted;
const heldRead = new Promise((resolve) => { releaseRead = resolve; });
const readStarted = new Promise((resolve) => { signalReadStarted = resolve; });
fsPromises.readFile.mockImplementationOnce(async (...args) => {
const bytes = await realFsPromises.readFile(...args);
signalReadStarted();
await heldRead;
return bytes;
});
const staleRead = store.load();
await readStarted;
try {
await store.mutate((data) => { data.items.push('first'); });
releaseRead();
await expect(staleRead).resolves.toEqual({ items: ['first'] });
await store.mutate((data) => { data.items.push('second'); });
expect(JSON.parse(await readFile(file, 'utf-8'))).toEqual({ items: ['first', 'second'] });
} finally {
releaseRead();
restoreFsMocks();
}
});
it('re-reads after an in-flight stale read races a persist and cache invalidation', async () => {
const file = join(dir, 'invalidated-state.json');
const store = createCachedStore(file, { items: [] }, { ttl: 60_000 });
await store.save({ items: [] });
store.invalidateCache();
let releaseRead;
let signalReadStarted;
const heldRead = new Promise((resolve) => { releaseRead = resolve; });
const readStarted = new Promise((resolve) => { signalReadStarted = resolve; });
fsPromises.readFile.mockImplementationOnce(async (...args) => {
const bytes = await realFsPromises.readFile(...args);
signalReadStarted();
await heldRead;
return bytes;
});
const staleRead = store.load();
await readStarted;
try {
await store.mutate((data) => { data.items.push('first'); });
store.invalidateCache();
releaseRead();
await expect(staleRead).resolves.toEqual({ items: ['first'] });
await store.mutate((data) => { data.items.push('second'); });
expect(JSON.parse(await readFile(file, 'utf-8'))).toEqual({ items: ['first', 'second'] });
} finally {
releaseRead();
restoreFsMocks();
}
});
it('mutate that returns undefined persists the mutated input', async () => {
const store = createCachedStore(join(dir, 'x.json'), { items: [] });
await store.mutate((data) => { data.items.push('a'); });
await store.mutate((data) => { data.items.push('b'); });
expect((await store.load()).items).toEqual(['a', 'b']);
});
it('keeps a rejected disk write out of cached reads and the next queued mutation', async () => {
const file = join(dir, 'accounts.json');
const store = createCachedStore(file, {}, { ttl: 60_000 });
const original = { accounts: { example: { status: 'pending' } }, count: 0 };
await store.save(structuredClone(original));
const cached = await store.load();
const diskError = Object.assign(new Error('synthetic replacement failure'), { code: 'EIO' });
vi.mocked(fsPromises.rename).mockRejectedValueOnce(diskError);
const rejected = store.mutate((data) => { delete data.accounts.example; });
const rejection = expect(rejected).rejects.toBe(diskError);
const next = store.mutate((data) => {
expect(data.accounts.example).toEqual({ status: 'pending' });
data.count += 1;
});
await rejection;
await next;
expect(cached).toEqual(original);
const expected = { accounts: { example: { status: 'pending' } }, count: 1 };
expect(await store.load()).toEqual(expected);
expect(JSON.parse(await readFile(file, 'utf-8'))).toEqual(expected);
});
it('hides an in-flight edit and discards it if the mutation callback rejects', async () => {
const file = join(dir, 'accounts.json');
const store = createCachedStore(file, {}, { ttl: 60_000 });
const original = { accounts: { example: { status: 'pending' } }, count: 0 };
await store.save(structuredClone(original));
let signalStarted;
let rejectEdit;
const started = new Promise((resolve) => { signalStarted = resolve; });
const editing = new Promise((_resolve, reject) => { rejectEdit = reject; });
const failed = store.mutate(async (data) => {
data.accounts.example.status = 'uncommitted';
signalStarted();
await editing;
});
const callbackError = new Error('synthetic callback failure');
const rejection = expect(failed).rejects.toBe(callbackError);
await started;
const duringEdit = await store.load();
rejectEdit(callbackError);
await rejection;
expect(duringEdit).toEqual(original);
expect(await store.load()).toEqual(original);
expect(JSON.parse(await readFile(file, 'utf-8'))).toEqual(original);
await store.mutate((data) => { data.count += 1; });
expect(JSON.parse(await readFile(file, 'utf-8'))).toEqual({ ...original, count: 1 });
});
it('invalidateCache forces a re-read from disk', async () => {
const file = join(dir, 'x.json');
const store = createCachedStore(file, { n: 0 });
await store.load();
await writeFile(file, JSON.stringify({ n: 42 }));
store.invalidateCache();
expect((await store.load()).n).toBe(42);
});
// Strict-read regression (#4115). Every consumer (accounts, feeds, schedules,
// personalities) mutates through `mutate`, which persists whatever `load`
// returned — so a corrupt file that read as the default was rewritten as the
// default on the next mutation, deleting every record it held. Corrupt JSON is
// the portable "present but unreadable" (same `ok: false` branch as EACCES).
describe('strict reads (#4115)', () => {
const CORRUPT = '{"accounts": {"a": 1},';
it('load rejects on a corrupt file instead of fabricating the default', async () => {
const file = join(dir, 'x.json');
await writeFile(file, CORRUPT);
const store = createCachedStore(file, { accounts: {} }, { context: 'accounts' });
await expect(store.load()).rejects.toThrow(/Unreadable JSON file/);
});
it('mutate leaves a corrupt file byte-for-byte intact', async () => {
const file = join(dir, 'x.json');
await writeFile(file, CORRUPT);
const store = createCachedStore(file, { accounts: {} });
await expect(store.mutate((data) => { data.accounts.b = 2; })).rejects.toThrow(/Unreadable JSON file/);
expect(await readFile(file, 'utf-8'), 'the default must never be written over the file we could not read').toBe(CORRUPT);
});
it('recovers once the file is repaired', async () => {
const file = join(dir, 'x.json');
await writeFile(file, CORRUPT);
const store = createCachedStore(file, { accounts: {} });
await expect(store.load()).rejects.toThrow(/Unreadable JSON file/);
await writeFile(file, JSON.stringify({ accounts: { a: 1 } }));
expect((await store.mutate((data) => { data.accounts.b = 2; })).accounts).toEqual({ a: 1, b: 2 });
});
});
});
describe('watchForFile', () => {
// Node 24's Windows libuv watcher can abort the worker in src/win/fs-event.c
// after a watched temp directory is removed, even after FSWatcher emits
// `close`. Linux covers the native integration; agentTuiSpawning tests cover
// the close-event and process-exit races with a platform-neutral watcher.
it.skipIf(process.platform === 'win32')('detects a newly written file with the event watcher and fallback poll', async () => {
const dir = mkdtempSync(join(tmpdir(), 'watch-file-test-'));
const target = join(dir, 'done.sentinel');
const detected = new Promise((resolve) => watchForFile(target, resolve, { settleMs: 0, pollMs: 10 }));
await new Promise((resolve) => setImmediate(resolve));
await writeFile(target, 'done');
await expect(detected).resolves.toBeUndefined();
// FSWatcher emits `close` before libuv's Windows endgame has fully released
// the directory handle. Let that native cleanup finish before removing the
// watched temp directory, or Node can abort in src/win/fs-event.c.
await new Promise((resolve) => setImmediate(resolve));
rmSync(dir, { recursive: true, force: true });
});
});
// =============================================================================
// sanitizeFilename / getFileExtension / getMimeType / allowlists (#1140)
// =============================================================================
describe('sanitizeFilename', () => {
it('returns a safe basename for a normal filename', () => {
expect(sanitizeFilename('hello.txt')).toBe('hello.txt');
});
it('strips directory components', () => {
expect(sanitizeFilename('/etc/passwd')).toBe('passwd');
expect(sanitizeFilename('../../etc/passwd')).toBe('passwd');
});
it('replaces special characters with underscores', () => {
expect(sanitizeFilename('hello world!.txt')).toBe('hello_world_.txt');
expect(sanitizeFilename('file with spaces.js')).toBe('file_with_spaces.js');
expect(sanitizeFilename('résumé.pdf')).toBe('r_sum_.pdf');
});
it('prevents hidden files by replacing a leading dot', () => {
expect(sanitizeFilename('.bashrc')).toBe('_bashrc');
expect(sanitizeFilename('.hidden.txt')).toBe('_hidden.txt');
});
it('leaves dots in the middle alone (extension dots)', () => {
expect(sanitizeFilename('my.file.name.txt')).toBe('my.file.name.txt');
});
it('handles an empty string gracefully', () => {
expect(sanitizeFilename('')).toBe('');
});
});
describe('getFileExtension', () => {
it('returns the lowercased extension with leading dot', () => {
expect(getFileExtension('photo.PNG')).toBe('.png');
expect(getFileExtension('archive.tar.gz')).toBe('.gz');
});
it('returns null for a filename without an extension', () => {
expect(getFileExtension('Makefile')).toBeNull();
expect(getFileExtension('no-ext')).toBeNull();
});
it('normalises uppercase extensions', () => {
expect(getFileExtension('IMAGE.JPEG')).toBe('.jpeg');
});
});
describe('getMimeType', () => {
it('returns the correct MIME type for known extensions', () => {
expect(getMimeType('.png')).toBe('image/png');
expect(getMimeType('.pdf')).toBe('application/pdf');
expect(getMimeType('.mp3')).toBe('audio/mpeg');
expect(getMimeType('.zip')).toBe('application/zip');
expect(getMimeType('.json')).toBe('application/json');
});
it('falls back to application/octet-stream for unknown extensions', () => {
expect(getMimeType('.xyz')).toBe('application/octet-stream');
expect(getMimeType('.unknown')).toBe('application/octet-stream');
expect(getMimeType(null)).toBe('application/octet-stream');
});
it('all entries in EXTENSION_MIME_MAP are reachable through getMimeType', () => {
for (const [ext, mime] of Object.entries(EXTENSION_MIME_MAP)) {
expect(getMimeType(ext)).toBe(mime);
}
});
});
describe('ATTACHMENT_ALLOWED_EXTENSIONS', () => {
it('is a Set', () => {
expect(ATTACHMENT_ALLOWED_EXTENSIONS).toBeInstanceOf(Set);
});
it('allows common document types', () => {
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.pdf')).toBe(true);
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.md')).toBe(true);
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.json')).toBe(true);
});
it('allows image types', () => {
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.png')).toBe(true);
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.jpg')).toBe(true);
});
it('does NOT allow audio/video types (strict attachment subset)', () => {
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.mp3')).toBe(false);
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.mp4')).toBe(false);
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.ico')).toBe(false);
expect(ATTACHMENT_ALLOWED_EXTENSIONS.has('.bmp')).toBe(false);
});
it('all extensions in the set are also present in EXTENSION_MIME_MAP', () => {
for (const ext of ATTACHMENT_ALLOWED_EXTENSIONS) {
expect(EXTENSION_MIME_MAP).toHaveProperty(ext);
}
});
});
describe('detectImageFormat', () => {
const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00]);
const JPEG = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);
const WEBP = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50]);
const GIF87 = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x37, 0x61, 0x00]);
const GIF89 = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x00]);
it('detects PNG', () => {
expect(detectImageFormat(PNG)).toEqual({ format: 'png', ext: '.png', mime: 'image/png' });
});
it('detects JPEG', () => {
expect(detectImageFormat(JPEG)).toEqual({ format: 'jpeg', ext: '.jpg', mime: 'image/jpeg' });
});
it('detects WebP', () => {
expect(detectImageFormat(WEBP)).toEqual({ format: 'webp', ext: '.webp', mime: 'image/webp' });
});
it('detects both GIF variants', () => {
expect(detectImageFormat(GIF87)?.format).toBe('gif');
expect(detectImageFormat(GIF89)?.format).toBe('gif');
});
it('returns null for non-image bytes', () => {
expect(detectImageFormat(Buffer.from('not an image'))).toBeNull();
});
it('returns null for a RIFF container that is not WebP', () => {
// RIFF header but "AVI " instead of "WEBP" at offset 8.
const avi = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x41, 0x56, 0x49, 0x20]);
expect(detectImageFormat(avi)).toBeNull();
});
it('returns null for a truncated/too-short buffer', () => {
expect(detectImageFormat(Buffer.from([0x89, 0x50]))).toBeNull();
});
it('returns null for a non-Buffer input', () => {
expect(detectImageFormat('AAAA')).toBeNull();
expect(detectImageFormat(null)).toBeNull();
});
});
describe('saveImageUpload (shared image upload pipeline)', () => {
let dir;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'save-image-test-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
// Exactly MIN_IMAGE_BYTES (12) — the shortest payload the floor accepts.
const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d]);
const b64 = (buf) => buf.toString('base64');
const opts = { maxBytes: 1024 };
it('persists the bytes verbatim and reports the detected format', async () => {
const saved = await saveImageUpload(dir, { filename: 'photo.png', data: b64(PNG) }, opts);
expect(saved.filename).toBe('photo.png');
expect(saved.size).toBe(12);
expect(saved.format).toBe('png');
expect(saved.mime).toBe('image/png');
expect(readFileSync(saved.filePath)).toEqual(PNG);
});
it('rejects a payload over maxBytes with a 400 FILE_TOO_LARGE', async () => {
await expect(saveImageUpload(dir, { filename: 'photo.png', data: b64(Buffer.alloc(2048)) }, opts))
.rejects.toMatchObject({ status: 400, code: 'FILE_TOO_LARGE' });
});
it('rejects bytes with no recognized image signature', async () => {
await expect(saveImageUpload(dir, { filename: 'photo.png', data: b64(Buffer.from('not an image!!')) }, opts))
.rejects.toMatchObject({ status: 400, code: 'INVALID_FILE_TYPE' });
});
// The per-format signatures are short (JPEG is 3 bytes, GIF 6) — without an
// explicit floor a truncated payload "detects" fine and saves as a success,
// handing the consumer a path to an unreadable file.
it('rejects a truncated payload whose signature alone would match', async () => {
for (const truncated of [
Buffer.from([0xff, 0xd8, 0xff]), // 3-byte JPEG
Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]), // 6-byte GIF
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), // 8-byte PNG
]) {
await expect(saveImageUpload(dir, { filename: 'photo.png', data: b64(truncated) }, opts))
.rejects.toMatchObject({ status: 400, code: 'INVALID_FILE_TYPE' });
}
expect(readdirSync(dir)).toHaveLength(0);
});
it('forces the extension to the detected format, not the claimed one', async () => {
const saved = await saveImageUpload(dir, { filename: 'photo.jpg', data: b64(PNG) }, opts);
expect(saved.filename).toBe('photo.jpg.png');
});
it('replaces a matching extension case-insensitively instead of doubling it', async () => {
const saved = await saveImageUpload(dir, { filename: 'PHOTO.PNG', data: b64(PNG) }, opts);
expect(saved.filename).toBe('PHOTO.png');
});
// A caller that prefixes the name can otherwise push a legitimately-long client
// filename past NAME_MAX and turn the write into an ENAMETOOLONG 500.
it('clamps the stored name to NAME_MAX, keeping the extension', async () => {
const saved = await saveImageUpload(dir, { filename: `${'a'.repeat(300)}.png`, data: b64(PNG) }, opts);
expect(saved.filename.length).toBe(255);
expect(saved.filename.endsWith('.png')).toBe(true);
expect(existsSync(saved.filePath)).toBe(true);
});
// #9721: the shared bucket's earlier images are referenced by task records, so
// a failed replacement must leave the prior bytes intact and no stage behind.
describe('publication failures (#9721)', () => {
const OLD = Buffer.concat([PNG, Buffer.from('previously stored image bytes')]);
const NEW = Buffer.concat([PNG, Buffer.from('replacement')]);
it('keeps the previous image intact when the write fails mid-way, then a retry succeeds', async () => {
writeFileSync(join(dir, 'diagram.png'), OLD);
const enospc = Object.assign(new Error('no space left on device'), { code: 'ENOSPC' });
fsPromises.writeFile.mockImplementationOnce(async (path) => {
await fsPromises.appendFile(path, NEW.subarray(0, 5)); // partial bytes land, then the write rejects
throw enospc;
});
await expect(saveImageUpload(dir, { filename: 'diagram.png', data: b64(NEW) }, { maxBytes: 4096 }))
.rejects.toBe(enospc);
expect(readFileSync(join(dir, 'diagram.png'))).toEqual(OLD);
expect(readdirSync(dir)).toEqual(['diagram.png']);
const saved = await saveImageUpload(dir, { filename: 'diagram.png', data: b64(NEW) }, { maxBytes: 4096 });
expect(readFileSync(saved.filePath)).toEqual(NEW);
expect(readdirSync(dir)).toEqual(['diagram.png']);
});
it('keeps the previous image and removes the stage when the rename fails', async () => {
writeFileSync(join(dir, 'diagram.png'), OLD);
const eio = Object.assign(new Error('io error'), { code: 'EIO' });
fsPromises.rename.mockImplementationOnce(async () => { throw eio; });
await expect(saveImageUpload(dir, { filename: 'diagram.png', data: b64(NEW) }, { maxBytes: 4096 }))
.rejects.toBe(eio);
expect(readFileSync(join(dir, 'diagram.png'))).toEqual(OLD);
expect(readdirSync(dir)).toEqual(['diagram.png']);
});
it('leaves a stage that carries the owner prefix and image extension when cleanup also fails', async () => {
const prefix = 'mind-11111111-2222-3333-4444-555555555555-';
const eio = Object.assign(new Error('io error'), { code: 'EIO' });
fsPromises.rename.mockImplementationOnce(async () => { throw eio; });
const unlinkSpy = vi.spyOn(fsPromises, 'unlink').mockRejectedValueOnce(new Error('unlink denied'));
await expect(saveImageUpload(dir, { filename: `${prefix}a.png`, data: b64(NEW) }, { maxBytes: 4096, stagePrefix: prefix }))
.rejects.toBe(eio);
unlinkSpy.mockRestore();
const [left, ...rest] = readdirSync(dir);
expect(rest).toEqual([]);
expect(left.startsWith(prefix)).toBe(true);
expect(left.endsWith('.png')).toBe(true);
});
it('stages with a bounded name so a NAME_MAX-length final name still publishes', async () => {
const saved = await saveImageUpload(dir, { filename: `${'a'.repeat(300)}.png`, data: b64(NEW) }, { maxBytes: 4096 });
expect(saved.filename.length).toBe(255);
expect(readFileSync(saved.filePath)).toEqual(NEW);
expect(readdirSync(dir)).toEqual([saved.filename]);
});
});
it('strips directory components from a traversal-shaped filename', async () => {
const saved = await saveImageUpload(dir, { filename: '../../evil.png', data: b64(PNG) }, opts);
expect(saved.filename).toBe('evil.png');
expect(saved.filePath.startsWith(dir)).toBe(true);
});
});
describe('saveBase64Upload (shared attachment upload pipeline)', () => {
let dir;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'save-b64-test-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
const opts = { allowedExtensions: SONGBOOK_ATTACHMENT_EXTENSIONS, maxBytes: 10 };
const b64 = (s) => Buffer.from(s).toString('base64');
it('persists within the cap and returns the uuid-prefixed name, buffer, size, and mime', async () => {
const saved = await saveBase64Upload(dir, { filename: 'sheet.txt', data: b64('ten bytes!') }, opts);
expect(saved.filename).toMatch(/^[0-9a-f]{8}-sheet\.txt$/);
expect(saved.size).toBe(10);
expect(saved.mime).toBe('text/plain');
expect(saved.buffer.toString('utf-8')).toBe('ten bytes!');
expect(existsSync(saved.filePath)).toBe(true);
expect(saved.filePath.startsWith(dir)).toBe(true);
});
it('rejects a payload over maxBytes with a 400 FILE_TOO_LARGE', async () => {
await expect(saveBase64Upload(dir, { filename: 'big.txt', data: b64('eleven bytes') }, opts))
.rejects.toMatchObject({ status: 400, code: 'FILE_TOO_LARGE' });
});
it('rejects an extension outside the allowlist with a 400 INVALID_FILE_TYPE', async () => {
await expect(saveBase64Upload(dir, { filename: 'app.exe', data: b64('x') }, opts))
.rejects.toMatchObject({ status: 400, code: 'INVALID_FILE_TYPE' });
});
it('rejects a traversal-shaped filename ("../x") with a 400', async () => {
await expect(saveBase64Upload(dir, { filename: '../x', data: b64('x') }, opts))
.rejects.toMatchObject({ status: 400 });
});
it('sanitizes a traversal filename with an allowed extension to a basename inside the dir', async () => {
const saved = await saveBase64Upload(dir, { filename: '../../evil.txt', data: b64('x') }, opts);
expect(saved.filename).toMatch(/^[0-9a-f]{8}-evil\.txt$/); // directory parts stripped
expect(saved.filePath.startsWith(dir)).toBe(true);
expect(existsSync(saved.filePath)).toBe(true);
});
});
describe('serveLocalFile (shared attachment serving pipeline)', () => {
let dir;
beforeEach(() => {
dir = mkdtempSync(join(tmpdir(), 'serve-local-test-'));
});
afterEach(() => {
rmSync(dir, { recursive: true, force: true });
});
const mockRes = () => ({
headers: {},
set(name, value) { this.headers[name] = value; return this; },
type: vi.fn(function type() { return this; }),
sendFile: vi.fn(),
});
it('serves a benign MIME inline with nosniff and no attachment disposition', async () => {
writeFileSync(join(dir, 'safe.txt'), 'hello');
const res = mockRes();
await serveLocalFile(res, dir, 'safe.txt');
expect(res.headers['X-Content-Type-Options']).toBe('nosniff');
expect(res.headers['Content-Disposition']).toBeUndefined();
expect(res.sendFile).toHaveBeenCalledWith(join(dir, 'safe.txt'));
});
it('forces Content-Disposition: attachment for a risky MIME (svg)', async () => {
writeFileSync(join(dir, 'sheet.svg'), '<svg/>');
const res = mockRes();
await serveLocalFile(res, dir, 'sheet.svg');
expect(res.headers['X-Content-Type-Options']).toBe('nosniff');
expect(res.headers['Content-Disposition']).toBe('attachment; filename="sheet.svg"');
expect(res.sendFile).toHaveBeenCalledWith(join(dir, 'sheet.svg'));
});
it('404s with the parametrized missingError for absent bytes', async () => {
const res = mockRes();
await expect(serveLocalFile(res, dir, 'nope.txt', {
missingError: { message: 'Attachment file is not on this machine', code: 'NOT_ON_THIS_MACHINE' },
})).rejects.toMatchObject({ status: 404, code: 'NOT_ON_THIS_MACHINE' });
expect(res.sendFile).not.toHaveBeenCalled();
});
// The declared `contentType` override (Beeper attachment mirror, #37/SEC-1):
// a remote sender's declared type is normalized (case folded, parameters
// dropped) and the disposition decision is taken on BOTH the declared value
// and whatever the type is bounded down to — so downgrading a risky
// `text/html` to `application/octet-stream` never costs it the attachment
// header it earned.
it.each([
['text/HTML'],
['TEXT/html'],
[' text/html ;charset=utf-8'],
['Image/SVG+xml'],
['application/xhtml+xml'],
['text/xml'],
])('forces Content-Disposition: attachment for a contentType override of %j', async (contentType) => {
writeFileSync(join(dir, 'override.bin'), 'x');
const res = mockRes();
await serveLocalFile(res, dir, 'override.bin', { contentType });
expect(res.headers['Content-Disposition']).toBe('attachment; filename="override.bin"');
});
it('bounds a contentType override outside the mirrored set to application/octet-stream instead of echoing it', async () => {
writeFileSync(join(dir, 'mystery.bin'), 'x');
const res = mockRes();
await serveLocalFile(res, dir, 'mystery.bin', { contentType: 'application/x-not-a-mirrored-type' });
expect(res.type).toHaveBeenCalledWith('application/octet-stream');
// Not risky, and the bounded type isn't either — no disposition override.
expect(res.headers['Content-Disposition']).toBeUndefined();
});
});