Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

7.1 KB
Raw Blame
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import fs from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { scanModelDuplicates, rectifyModelDuplicates } from './modelDeduplication.js';

vi.mock('./lmStudioManager.js', () => ({ getModelsDir: async () => process.env.LM_STUDIO_MODELS_DIR }));

let root;
let roots;
const bytes = Buffer.alloc(10 * 1024 * 1024, 7);
async function weight(dir, name = 'example.safetensors', data = bytes) {
  await fs.mkdir(dir, { recursive: true });
  const path = join(dir, name);
  await fs.writeFile(path, data);
  return path;
}
beforeEach(async () => {
  root = await fs.realpath(await fs.mkdtemp(join(tmpdir(), 'model-dedupe-')));
  roots = { local: [join(root, 'local')], external: [join(root, 'pinokio')] };
  await Promise.all([...roots.local, ...roots.external].map((dir) => fs.mkdir(dir)));
});
afterEach(async () => {
  vi.restoreAllMocks();
  vi.unstubAllEnvs();
  await fs.rm(root, { recursive: true, force: true });
});

it('verifies contents, preserves HF snapshot links and links multiple copies to one source', async () => {
  const blob = await weight(join(roots.local[0], 'blobs'), 'hash');
  const snapshot = join(roots.local[0], 'snapshots', 'revision');
  await fs.mkdir(snapshot, { recursive: true });
  const sourcePath = join(snapshot, 'example.safetensors');
  await fs.symlink(blob, sourcePath);
  const targetPath = await weight(join(roots.external[0], 'first'));
  const secondPath = await weight(join(roots.external[0], 'second'));
  const wrongPath = await weight(join(roots.external[0], 'wrong'), 'example.safetensors', Buffer.alloc(bytes.length, 8));
  const report = await scanModelDuplicates({ roots });
  expect(report.items).toHaveLength(2);
  expect(report.items[0].sizeBytes).toBe(bytes.length);
  expect(report.totalReclaimableBytes).toBe((await fs.stat(targetPath)).blocks * 1024);
  const rename = vi.spyOn(fs, 'rename');
  const result = await rectifyModelDuplicates(report.items, { roots });
  expect(result.reclaimedBytes).toBe(report.totalReclaimableBytes);
  expect(rename).toHaveBeenCalledTimes(2);
  expect((await fs.lstat(sourcePath)).isSymbolicLink()).toBe(true);
  for (const path of [targetPath, secondPath]) {
    expect((await fs.stat(path)).ino).toBe((await fs.stat(blob)).ino);
    expect((await fs.readFile(path)).equals(bytes)).toBe(true);
  }
  expect((await fs.readFile(wrongPath)).equals(Buffer.alloc(bytes.length, 8))).toBe(true);
  const rescanned = await scanModelDuplicates({ roots });
  expect(rescanned.totalReclaimableBytes).toBe(0);
  expect(rescanned.items.every((item) => item.alreadyLinked)).toBe(true);
});

it('rejects mismatches and symlink escapes before replacing any batch entry', async () => {
  const sourcePath = await weight(roots.local[0]);
  const targetPath = await weight(roots.external[0]);
  const original = (await fs.stat(targetPath)).ino;
  const outside = await weight(root, 'outside.safetensors');
  const escape = join(roots.external[0], 'escape.safetensors');
  await fs.symlink(outside, escape);
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath }, { sourcePath, targetPath: escape }], { roots })).rejects.toThrow('Unsafe');
  expect((await fs.stat(targetPath)).ino).toBe(original);
  await fs.writeFile(targetPath, Buffer.alloc(bytes.length, 9));
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath }], { roots })).rejects.toThrow('contents differ');
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath: outside }], { roots })).rejects.toThrow('outside');
});

it('does not count target aliases and rejects cross-device replacement', async () => {
  const sourcePath = await weight(roots.local[0]);
  const targetPath = await weight(roots.external[0]);
  const alias = join(roots.external[0], 'alias.safetensors');
  await fs.link(targetPath, alias);
  expect((await scanModelDuplicates({ roots })).totalReclaimableBytes).toBe(0);
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath }], { roots })).rejects.toThrow('additional hardlinks');
  await fs.unlink(alias);
  const stat = fs.stat.bind(fs);
  vi.spyOn(fs, 'stat').mockImplementation(async (path, ...args) => {
    const info = await stat(path, ...args);
    if (path === targetPath) info.dev += 1n;
    return info;
  });
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath }], { roots })).rejects.toThrow('span filesystems');
});

it('cleans a failed rename without replacing the original', async () => {
  const sourcePath = await weight(roots.local[0]);
  const targetPath = await weight(roots.external[0]);
  const original = (await fs.stat(targetPath)).ino;
  vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed'));
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath }], { roots })).rejects.toThrow('rename failed');
  expect((await fs.stat(targetPath)).ino).toBe(original);
  expect(await fs.readdir(roots.external[0])).toEqual(['example.safetensors']);
});

it('does no directory traversal when Pinokio is absent', async () => {
  vi.stubEnv('PINOKIO_HOME', join(root, 'absent'));
  const read = vi.spyOn(fs, 'readdir');
  expect(await scanModelDuplicates()).toEqual({ pinokioDetected: false, items: [], totalReclaimableBytes: 0 });
  expect(read).not.toHaveBeenCalled();
});

it('discovers peer drives, app models and the separate HF cache from configured roots', async () => {
  vi.stubEnv('PINOKIO_HOME', roots.external[0]);
  vi.stubEnv('HF_HUB_CACHE', roots.local[0]);
  vi.stubEnv('LM_STUDIO_MODELS_DIR', join(root, 'absent-lm'));
  await weight(roots.local[0]);
  await weight(join(roots.external[0], 'drive', 'drives', 'peers', 'example-peer', 'mlx_models', 'example-model'));
  await weight(join(roots.external[0], 'api', 'example-app', 'models'));
  await weight(join(roots.external[0], 'cache', 'huggingface', 'hub', 'models--example--model', 'snapshots', 'revision'));
  const report = await scanModelDuplicates();
  expect(report.pinokioDetected).toBe(true);
  expect(report.items).toHaveLength(3);
});

// Windows inode identifiers can exceed Number's exact integer range. Distinct
// files must still reach hash verification and atomic replacement.
it('does not mistake rounded inode identifiers for already-linked files', async () => {
  const sourcePath = await weight(roots.local[0]);
  const targetPath = await weight(roots.external[0]);
  const stat = fs.stat.bind(fs);
  const sourceInode = 2n ** 60n;
  const targetInode = sourceInode + 1n;
  expect(Number(sourceInode)).toBe(Number(targetInode));
  vi.spyOn(fs, 'stat').mockImplementation(async (path, options) => {
    const info = await stat(path, options);
    if (path === sourcePath || path === targetPath) {
      const inode = path === sourcePath ? sourceInode : targetInode;
      info.ino = options?.bigint ? inode : Number(inode);
    }
    return info;
  });
  vi.spyOn(fs, 'rename').mockRejectedValue(new Error('rename failed'));
  await expect(rectifyModelDuplicates([{ sourcePath, targetPath }], { roots })).rejects.toThrow('rename failed');
  expect(await fs.readdir(roots.external[0])).toEqual(['example.safetensors']);
  expect((await fs.readFile(targetPath)).equals(bytes)).toBe(true);
});