Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

39.6 KB
Raw Blame
/**
 * Boundary tests for the Beeper attachment byte mirror (#37).
 *
 * Postgres is mocked at `query()` and the Beeper client at `headAsset` /
 * `fetchAssetStream`, but the FILESYSTEM is real (a temp `PATHS.data`): the
 * behaviour under test is what lands on disk and what the store does when
 * asked to give bytes back, and a mocked `fs` would only prove the mock. The
 * behavioural acceptance path against a real database is not covered here —
 * these are the byte-level contracts.
 *
 * Every fixture value is invented (placeholder ids, `example` names) per root
 * AGENTS.md Sensitive Data & Privacy — no value here came from a running
 * instance, and nothing here sends anything through a messaging API.
 */
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
import { mkdirSync, readdirSync, rmSync, utimesSync, writeFileSync, readFileSync, existsSync } from 'fs';
import { dirname, join } from 'path';
import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';

const tempRoot = createTempDataRoot('portos-beeper-attachments-');

vi.mock('../lib/db.js', () => ({ query: vi.fn() }));
const publicationSeam = vi.hoisted(() => ({ bypass: false, reached: null }));
vi.mock('../lib/backupSnapshotBoundary.js', async original => {
  const actual = await original();
  return { ...actual, withBackupAssetPublication: work => {
    publicationSeam.reached?.();
    return publicationSeam.bypass ? work() : actual.withBackupAssetPublication(work);
  } };
});

vi.mock('./settings.js', () => ({ getSettings: vi.fn() }));
// The client is mocked WHOLE rather than partially: importing the real module
// pulls in the vault-backed credential store, which reads `PATHS` at import
// time — before the hoisted temp-root mock below can answer. The service only
// ever branches on `err.code === 'ASSET_UNAVAILABLE'`, and that mapping (a 502
// from `serve` for media the network aged out) is pinned where it is decided,
// in `beeperClient.test.js`.
vi.mock('./beeperClient.js', () => {
  class MockBeeperApiError extends Error {
    constructor(message, { status = 500, code, retryable = false } = {}) {
      super(message);
      this.name = 'BeeperApiError';
      this.status = status;
      this.code = code;
      this.retryable = retryable;
    }
  }
  return { BeeperApiError: MockBeeperApiError, headAsset: vi.fn(), fetchAssetStream: vi.fn() };
});
vi.mock('../lib/fileUtils.js', async (importOriginal) =>
  makePathsProxy(await importOriginal(), { dataRoot: () => tempRoot }));

import { query } from '../lib/db.js';
import { getSettings } from './settings.js';
import { BeeperApiError, headAsset, fetchAssetStream } from './beeperClient.js';
import {
  attachmentsRoot,
  backfillAttachments,
  ensureAttachmentBytes,
  evictToBudget,
  getAttachment,
  getAttachmentSummary,
  purgeConversationAttachments,
  shapeAttachment,
  sweepAttachmentOrphans,
} from './beeperAttachments.js';

const CONV = '11111111-1111-4111-8111-111111111111';

const attachmentRow = (overrides = {}) => ({
  conversation_id: CONV,
  message_id: 'msg-example-1',
  idx: 0,
  mxc_id: 'mxc://example.invalid/abc',
  mime_type: 'image/png',
  byte_length: null,
  file_name: 'example.png',
  width: null,
  height: null,
  last_viewed_at: null,
  keep: false,
  local_path: null,
  fetched_at: null,
  unavailable_at: null,
  fetch_error: null,
  ...overrides,
});

// A `Response`-shaped fake whose `body` is a real web ReadableStream, because
// the mirror consumes it through `Readable.fromWeb` — a fake array would not
// exercise the byte counter that enforces the mid-stream ceiling.
const streamResponse = (chunks) => ({
  ok: true,
  status: 200,
  body: new ReadableStream({
    start(controller) {
      for (const chunk of chunks) controller.enqueue(chunk);
      controller.close();
    },
  }),
});

const unavailableError = () => new BeeperApiError('Failed to download asset: Transfer failed', {
  status: 502, code: 'ASSET_UNAVAILABLE', retryable: false,
});

/**
 * Answer `query()` from a table of SQL fragment → result, so a test states the
 * three or four statements it cares about instead of ordering every call.
 */
const respondTo = (table, fallback = { rows: [] }) => {
  vi.mocked(query).mockImplementation(async (sql) => {
    const flat = String(sql).replace(/\s+/g, ' ');
    for (const [fragment, result] of table) {
      if (flat.includes(fragment)) return typeof result === 'function' ? result(flat) : result;
    }
    return fallback;
  });
};

const storedFiles = () => {
  const root = attachmentsRoot();
  if (!existsSync(root)) return [];
  return readdirSync(root, { withFileTypes: true })
    .filter((entry) => entry.isDirectory() && entry.name !== '.tmp')
    .flatMap((entry) => readdirSync(join(root, entry.name)).map((name) => `${entry.name}/${name}`));
};

beforeEach(() => {
  vi.clearAllMocks();
  publicationSeam.bypass = false; publicationSeam.reached = null;
  vi.mocked(getSettings).mockResolvedValue({ beeper: { attachmentBudgetGb: 5 } });
  rmSync(tempRoot, { recursive: true, force: true });
  mkdirSync(tempRoot, { recursive: true });
});

afterEach(() => rmSync(tempRoot, { recursive: true, force: true }));

describe('ensureAttachmentBytes — the lazy mirror', () => {
  it('downloads on first view, stores content-addressed, and serves from disk on the second', async () => {
    let row = attachmentRow();
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('FROM beeper_attachments WHERE message_id')) return { rows: [row] };
      if (flat.includes('SET local_path = $3')) {
        // No `sha256` column write (audit cluster 06 decision 3) — the UPDATE
        // is `local_path, byte_length`, so `params` is 4-long.
        row = { ...row, local_path: params[2], byte_length: params[3] };
        return { rows: [] };
      }
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 5 });
    vi.mocked(fetchAssetStream).mockResolvedValue(streamResponse([new Uint8Array([1, 2, 3, 4, 5])]));

    const first = await ensureAttachmentBytes('msg-example-1', 0);
    expect(first.cached).toBe(false);
    expect(first.mimeType).toBe('image/png');
    // `<sha256[0..2]>/<sha256>.png` — one file, addressed by content.
    expect(storedFiles()).toHaveLength(1);
    expect(storedFiles()[0]).toMatch(/^[0-9a-f]{2}\/[0-9a-f]{64}\.png$/);

    const second = await ensureAttachmentBytes('msg-example-1', 0);
    expect(second.cached).toBe(true);
    // The second open costs nothing upstream — no HEAD, no stream.
    expect(vi.mocked(fetchAssetStream)).toHaveBeenCalledTimes(1);
    expect(vi.mocked(headAsset)).toHaveBeenCalledTimes(1);
  });

  it('refuses an over-cap attachment at the HEAD pre-flight, before a byte moves', async () => {
    respondTo([['FROM beeper_attachments WHERE message_id', { rows: [attachmentRow()] }]]);
    vi.mocked(headAsset).mockResolvedValue({ bytes: 40 * 1024 * 1024 });

    await expect(ensureAttachmentBytes('msg-example-1', 0)).rejects.toMatchObject({
      code: 'ATTACHMENT_TOO_LARGE', status: 413, context: { bytes: 40 * 1024 * 1024, maxBytes: 32 * 1024 * 1024 },
    });
    expect(vi.mocked(fetchAssetStream)).not.toHaveBeenCalled();
    expect(storedFiles()).toEqual([]);
  });

  it('records the size the HEAD reported when it refuses, so the next render is the real placeholder', async () => {
    // A row the bridge never reported a size for is NOT `overCap`, so the
    // thread renders an <img>, takes the 413, and would otherwise fall into the
    // generic retry — whose retry forces an unbounded download behind a button
    // that names neither the size nor the ceiling.
    respondTo([['FROM beeper_attachments WHERE message_id', { rows: [attachmentRow({ byte_length: null })] }]]);
    vi.mocked(headAsset).mockResolvedValue({ bytes: 41 * 1024 * 1024 });

    await expect(ensureAttachmentBytes('msg-example-1', 0)).rejects.toMatchObject({ code: 'ATTACHMENT_TOO_LARGE' });
    const write = vi.mocked(query).mock.calls.find(([sql]) => String(sql).includes('SET byte_length = $3'));
    expect(write).toBeTruthy();
    expect(write[1]).toEqual(['msg-example-1', 0, 41 * 1024 * 1024]);
  });

  it('abandons a transfer that goes silent mid-stream instead of holding the file open', async () => {
    respondTo([['FROM beeper_attachments WHERE message_id', { rows: [attachmentRow()] }]]);
    vi.mocked(headAsset).mockResolvedValue({ bytes: 8 });
    // A body that emits one chunk and then never resolves again — the shape the
    // header-only fetch budget cannot see. The mirror's own idle abort is what
    // ends it; the signal it passes is the one under test.
    vi.mocked(fetchAssetStream).mockImplementation(async (_id, { signal } = {}) => ({
      ok: true,
      status: 200,
      body: new ReadableStream({
        start(controller) {
          controller.enqueue(new Uint8Array([1, 2, 3, 4]));
          signal?.addEventListener('abort', () => controller.error(new Error('aborted')), { once: true });
        },
      }),
    }));
    vi.useFakeTimers();
    const pending = ensureAttachmentBytes('msg-example-1', 0);
    const settled = pending.then(() => null).catch((err) => err);
    await vi.advanceTimersByTimeAsync(61_000);
    const failure = await settled;
    vi.useRealTimers();

    expect(failure).toMatchObject({ code: 'ATTACHMENT_DOWNLOAD_FAILED' });
    expect(storedFiles()).toEqual([]);
    expect(readdirSync(join(attachmentsRoot(), '.tmp'))).toEqual([]);
  });

  it('aborts mid-stream when the source declines to report a size and overruns the ceiling', async () => {
    respondTo([['FROM beeper_attachments WHERE message_id', { rows: [attachmentRow()] }]]);
    // `bytes: null` is "the server would not say" — which must not read as
    // zero and sail under the ceiling.
    vi.mocked(headAsset).mockResolvedValue({ bytes: null });
    const oversized = () => streamResponse(
      Array.from({ length: 34 }, () => new Uint8Array(1024 * 1024)),
    );
    vi.mocked(fetchAssetStream).mockImplementation(async () => oversized());

    await expect(ensureAttachmentBytes('msg-example-1', 0)).rejects.toMatchObject({
      code: 'ATTACHMENT_TOO_LARGE',
    });
    // Nothing finalized, and the partial is cleaned up rather than left to
    // count against the budget.
    expect(storedFiles()).toEqual([]);
    expect(readdirSync(join(attachmentsRoot(), '.tmp'))).toEqual([]);
  });

  it('lifts the ceiling only for an explicit "fetch anyway"', async () => {
    respondTo([['FROM beeper_attachments WHERE message_id', { rows: [attachmentRow({ byte_length: 40 * 1024 * 1024 })] }]]);
    vi.mocked(headAsset).mockResolvedValue({ bytes: 40 * 1024 * 1024 });
    vi.mocked(fetchAssetStream).mockResolvedValue(streamResponse([new Uint8Array(16)]));

    const result = await ensureAttachmentBytes('msg-example-1', 0, { force: true });
    expect(result.cached).toBe(false);
    expect(storedFiles()).toHaveLength(1);
  });

  it('stamps a 502 as terminal and never re-requests it on the next view', async () => {
    const calls = [];
    let row = attachmentRow();
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      calls.push(flat);
      if (flat.includes('FROM beeper_attachments WHERE message_id')) return { rows: [row] };
      if (flat.includes('SET unavailable_at = NOW()')) {
        row = { ...row, unavailable_at: '2026-09-03T00:00:00.000Z', fetch_error: params[2] };
        return { rows: [] };
      }
      return { rows: [] };
    });
    vi.mocked(headAsset).mockRejectedValue(unavailableError());

    await expect(ensureAttachmentBytes('msg-example-1', 0)).rejects.toMatchObject({ code: 'ASSET_UNAVAILABLE' });
    expect(calls.some((sql) => sql.includes('SET unavailable_at = NOW()'))).toBe(true);

    await expect(ensureAttachmentBytes('msg-example-1', 0)).rejects.toMatchObject({
      code: 'ASSET_UNAVAILABLE', status: 404,
    });
    // The second view short-circuits on the stamp — one probe, not a loop.
    expect(vi.mocked(headAsset)).toHaveBeenCalledTimes(1);
  });

  it('never turns a malformed stored path into a filesystem read — it re-acquires instead', async () => {
    // `local_path` is a DB column that becomes a path, and a mirror row may have
    // been written by any past version of PortOS. A value that is not this
    // layout's own `<sha256[0..2]>/<sha256>.<ext>` is refused as a path outright
    // rather than resolved and stat-ed.
    let row = attachmentRow({ local_path: '../../../etc/hosts' });
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('FROM beeper_attachments WHERE message_id')) return { rows: [row] };
      if (flat.includes('SET local_path = $3')) {
        row = { ...row, local_path: params[2] };
        return { rows: [] };
      }
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 2 });
    vi.mocked(fetchAssetStream).mockResolvedValue(streamResponse([new Uint8Array([1, 2])]));

    const result = await ensureAttachmentBytes('msg-example-1', 0);
    expect(result.filePath.startsWith(attachmentsRoot())).toBe(true);
    expect(result.filePath).not.toContain('etc/hosts');
  });

  it('heals a row whose file was removed from under it instead of 404ing', async () => {
    let row = attachmentRow({ local_path: `aa/${'a'.repeat(64)}.png`, fetched_at: '2026-09-01T00:00:00.000Z' });
    const cleared = [];
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('FROM beeper_attachments WHERE message_id')) return { rows: [row] };
      if (flat.includes('SET local_path = NULL')) {
        cleared.push(flat);
        row = { ...row, local_path: null };
        return { rows: [] };
      }
      if (flat.includes('SET local_path = $3')) return { rows: [] };
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 3 });
    vi.mocked(fetchAssetStream).mockResolvedValue(streamResponse([new Uint8Array([9, 9, 9])]));

    const result = await ensureAttachmentBytes('msg-example-1', 0);
    expect(cleared.length).toBe(1);
    expect(result.cached).toBe(false);
    expect(storedFiles()).toHaveLength(1);
  });
});

// LENS-3/PERF-2: the PK on beeper_attachments leads with `conversation_id`,
// which every one of these lookups is written WITHOUT — this pins the query
// shape (`message_id` leading, matching `idx_beeper_attachments_message`) so
// a future edit can't silently regress it back onto a table scan. The actual
// plan is proven against a real index in beeperConversations.db.test.js.
describe('attachment lookups lead with message_id, matching the new index', () => {
  it('getAttachment queries WHERE message_id = $1 AND idx = $2, in that column order', async () => {
    respondTo([['FROM beeper_attachments WHERE message_id', { rows: [attachmentRow()] }]]);
    await getAttachment('msg-example-1', 0);
    const call = vi.mocked(query).mock.calls.find(([sql]) => String(sql).includes('FROM beeper_attachments WHERE message_id'));
    expect(call).toBeTruthy();
    expect(String(call[0]).replace(/\s+/g, ' ')).toContain('WHERE message_id = $1 AND idx = $2');
  });
});

describe('evictToBudget — least-recently-viewed, guarded by a HEAD', () => {
  const relPath = `aa/${'a'.repeat(64)}.png`;

  const seedFile = (relativePath, bytes = 16) => {
    mkdirSync(join(attachmentsRoot(), relativePath.split('/')[0]), { recursive: true });
    writeFileSync(join(attachmentsRoot(), relativePath), Buffer.alloc(bytes));
  };

  it('never offers a row with no source reference as a candidate', async () => {
    respondTo([['unique_files', { rows: [{ bytes: String(10 * 1024 * 1024 * 1024), files: 1 }] }]]);
    await evictToBudget();
    // Eviction is only safe because the bytes can be fetched again; a row with
    // no `mxc_id` is precisely the one that could not be, so it is filtered in
    // the candidate query rather than skipped after selection (which would
    // re-select it forever).
    const candidateSql = vi.mocked(query).mock.calls
      .map(([sql]) => String(sql).replace(/\s+/g, ' '))
      .find((sql) => sql.includes('ORDER BY last_viewed_at ASC NULLS FIRST'));
    expect(candidateSql).toContain('mxc_id IS NOT NULL');
  });

  it('does nothing while the mirror fits its budget', async () => {
    respondTo([['unique_files', { rows: [{ bytes: '1024', files: 1 }] }]]);
    const result = await evictToBudget();
    expect(result).toMatchObject({ evicted: 0, overBudget: false });
    expect(vi.mocked(headAsset)).not.toHaveBeenCalled();
  });

  it('evicts the least-recently-viewed file once the source confirms it is re-fetchable', async () => {
    seedFile(relPath);
    let used = 10 * 1024 * 1024 * 1024;
    let candidateRemaining = true;
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('unique_files')) return { rows: [{ bytes: String(used), files: 1 }] };
      if (flat.includes('ORDER BY last_viewed_at ASC NULLS FIRST')) {
        return candidateRemaining
          ? { rows: [{ message_id: 'msg-example-1', idx: 0, mxc_id: 'mxc://example.invalid/abc', local_path: relPath, byte_length: 16 }] }
          : { rows: [] };
      }
      if (flat.includes('SET local_path = NULL')) { candidateRemaining = false; used = 0; return { rows: [] }; }
      if (flat.includes('WHERE local_path = $1 LIMIT 1')) return { rows: [] };
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 16 });

    const result = await evictToBudget();
    expect(result.evicted).toBe(1);
    expect(existsSync(join(attachmentsRoot(), relPath))).toBe(false);
  });

  // LENS-4: the candidate index now orders `last_viewed_at ASC NULLS FIRST` —
  // a never-viewed attachment is the BEST eviction candidate (least, not most,
  // recently seen), and this pins that the service still evicts it cleanly
  // when the query hands one back, rather than choking on the `null`.
  it('evicts a never-viewed candidate (NULL last_viewed_at) exactly like any other', async () => {
    seedFile(relPath);
    let candidateRemaining = true;
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('unique_files')) return { rows: [{ bytes: String(10 * 1024 * 1024 * 1024), files: 1 }] };
      if (flat.includes('ORDER BY last_viewed_at ASC NULLS FIRST')) {
        return candidateRemaining
          ? { rows: [{
            message_id: 'msg-example-1', idx: 0, mxc_id: 'mxc://example.invalid/abc',
            local_path: relPath, byte_length: 16, last_viewed_at: null,
          }] }
          : { rows: [] };
      }
      if (flat.includes('SET local_path = NULL')) { candidateRemaining = false; return { rows: [] }; }
      if (flat.includes('WHERE local_path = $1 LIMIT 1')) return { rows: [] };
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 16 });

    const result = await evictToBudget();
    expect(result.evicted).toBe(1);
    expect(existsSync(join(attachmentsRoot(), relPath))).toBe(false);
  });

  it('KEEPS a file the source can no longer supply, and takes it out of future candidate queries', async () => {
    seedFile(relPath);
    const marked = [];
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('unique_files')) return { rows: [{ bytes: String(10 * 1024 * 1024 * 1024), files: 1 }] };
      if (flat.includes('ORDER BY last_viewed_at ASC NULLS FIRST')) {
        return marked.length === 0
          ? { rows: [{ message_id: 'msg-example-1', idx: 0, mxc_id: 'mxc://example.invalid/abc', local_path: relPath, byte_length: 16 }] }
          : { rows: [] };
      }
      if (flat.includes('SET unavailable_at = NOW()')) { marked.push(flat); return { rows: [] }; }
      return { rows: [] };
    });
    vi.mocked(headAsset).mockRejectedValue(unavailableError());

    const result = await evictToBudget();
    expect(result).toMatchObject({ evicted: 0, keptUnavailable: 1 });
    expect(marked).toHaveLength(1);
    // The bytes Beeper can no longer supply are the last copy — still there.
    expect(existsSync(join(attachmentsRoot(), relPath))).toBe(true);
  });

  it('stops rather than evicting when the probe itself fails (Beeper closed)', async () => {
    seedFile(relPath);
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('unique_files')) return { rows: [{ bytes: String(10 * 1024 * 1024 * 1024), files: 1 }] };
      if (flat.includes('ORDER BY last_viewed_at ASC NULLS FIRST')) {
        return { rows: [{ message_id: 'msg-example-1', idx: 0, mxc_id: 'mxc://example.invalid/abc', local_path: relPath, byte_length: 16 }] };
      }
      return { rows: [] };
    });
    vi.mocked(headAsset).mockRejectedValue(new BeeperApiError('Beeper asset request failed: connection refused', {
      status: 0, code: 'NETWORK_ERROR', retryable: false,
    }));

    const result = await evictToBudget();
    expect(result).toMatchObject({ evicted: 0, overBudget: true });
    expect(existsSync(join(attachmentsRoot(), relPath))).toBe(true);
  });
});

describe('the store is content-addressed, so every unlink is reference-checked', () => {
  const relPath = `cc/${'c'.repeat(64)}.png`;

  it('keeps a forwarded file that another conversation still points at', async () => {
    mkdirSync(join(attachmentsRoot(), 'cc'), { recursive: true });
    writeFileSync(join(attachmentsRoot(), relPath), Buffer.alloc(32));
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('SELECT DISTINCT local_path FROM beeper_attachments WHERE conversation_id')) {
        return { rows: [{ local_path: relPath }] };
      }
      // A surviving row in ANOTHER conversation still points at these bytes.
      if (flat.includes('WHERE local_path = $1 LIMIT 1')) return { rows: [{ '?column?': 1 }] };
      return { rows: [] };
    });

    const result = await purgeConversationAttachments(CONV);
    expect(result).toEqual({ removedFiles: 0, freedBytes: 0, failedUnlinks: 0 });
    expect(existsSync(join(attachmentsRoot(), relPath))).toBe(true);
  });

  it('removes the file once nothing outside the purged conversation references it', async () => {
    mkdirSync(join(attachmentsRoot(), 'cc'), { recursive: true });
    writeFileSync(join(attachmentsRoot(), relPath), Buffer.alloc(32));
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('SELECT DISTINCT local_path FROM beeper_attachments WHERE conversation_id')) {
        return { rows: [{ local_path: relPath }] };
      }
      return { rows: [] };
    });

    const result = await purgeConversationAttachments(CONV);
    expect(result).toMatchObject({ removedFiles: 1, freedBytes: 32, failedUnlinks: 0 });
    expect(existsSync(join(attachmentsRoot(), relPath))).toBe(false);
  });

  it('counts an unlink it could not perform instead of reporting the bytes as reclaimed', async () => {
    // A directory standing where the mirrored file should be: `unlink` refuses
    // it, which is the shape every un-unlinkable path takes — a read-only
    // mount, a permissions change, a handle the OS will not let go of.
    const blocked = `ab/${'ab'.repeat(32)}.png`;
    mkdirSync(join(attachmentsRoot(), blocked), { recursive: true });
    respondTo([
      ['SELECT DISTINCT local_path FROM beeper_attachments WHERE conversation_id', { rows: [{ local_path: blocked }] }],
    ]);
    const errors = [];
    const errorSpy = vi.spyOn(console, 'error').mockImplementation((line) => { errors.push(String(line)); });

    const result = await purgeConversationAttachments(CONV);

    // The row has already given up its claim, so nothing points at these bytes
    // any more. Reporting them as reclaimed is exactly how they vanish from the
    // budget while still sitting on the disk.
    expect(result).toEqual({ removedFiles: 0, freedBytes: 0, failedUnlinks: 1 });
    expect(existsSync(join(attachmentsRoot(), blocked))).toBe(true);
    expect(errors.some((line) => line.includes('unlink failed'))).toBe(true);
    errorSpy.mockRestore();
  });
});

describe('sweepAttachmentOrphans — the backstop, both directions', () => {
  it('removes a file no row points at and heals a row whose file is gone', async () => {
    const orphan = `ee/${'e'.repeat(64)}.png`;
    const missing = `11/${'1'.repeat(64)}.png`;
    mkdirSync(join(attachmentsRoot(), 'ee'), { recursive: true });
    writeFileSync(join(attachmentsRoot(), orphan), Buffer.alloc(8));
    // Backdated past the sweep's age gate: a file linked into place seconds ago
    // may be a download whose row update has not landed yet, so only an old
    // unreferenced file counts as an orphan.
    const hoursAgo = new Date(Date.now() - 4 * 60 * 60 * 1000);
    utimesSync(join(attachmentsRoot(), orphan), hoursAgo, hoursAgo);

    const healed = [];
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('SELECT DISTINCT local_path FROM beeper_attachments WHERE local_path IS NOT NULL')) {
        return { rows: [{ local_path: missing }] };
      }
      if (flat.includes('WHERE local_path = ANY')) { healed.push(params[0]); return { rowCount: 1 }; }
      return { rows: [] };
    });

    const result = await sweepAttachmentOrphans();
    expect(result).toMatchObject({ orphansRemoved: 1, healedRows: 1 });
    expect(existsSync(join(attachmentsRoot(), orphan))).toBe(false);
    expect(healed[0]).toEqual([missing]);
  });

  it('never heals a row whose file landed after the listing, and fences the UPDATE on fetched_at', async () => {
    const landed = `cc/${'c'.repeat(64)}.png`;
    const gone = `11/${'1'.repeat(64)}.png`;
    // The download commits its file after listStoredFiles() but before the
    // reference query runs, so the file is referenced yet absent from the listing.
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('SELECT DISTINCT local_path FROM beeper_attachments WHERE local_path IS NOT NULL')) {
        mkdirSync(join(attachmentsRoot(), 'cc'), { recursive: true });
        writeFileSync(join(attachmentsRoot(), landed), Buffer.alloc(8));
        return { rows: [{ local_path: landed }, { local_path: gone }] };
      }
      if (flat.includes('WHERE local_path = ANY')) {
        expect(flat).toContain('fetched_at < $2');
        expect(params[1]).toBeInstanceOf(Date);
        return { rowCount: params[0].length };
      }
      return { rows: [] };
    });

    const result = await sweepAttachmentOrphans();
    expect(result.healedRows).toBe(1);
    const update = vi.mocked(query).mock.calls.find(([sql]) => String(sql).includes('WHERE local_path = ANY'));
    expect(update[1][0]).toEqual([gone]);
    expect(existsSync(join(attachmentsRoot(), landed))).toBe(true);
  });

  it('spares a file that gained a reference between the snapshot and the unlink', async () => {
    const shared = `ff/${'f'.repeat(64)}.png`;
    mkdirSync(join(attachmentsRoot(), 'ff'), { recursive: true });
    writeFileSync(join(attachmentsRoot(), shared), Buffer.alloc(8));
    const hoursAgo = new Date(Date.now() - 4 * 60 * 60 * 1000);
    utimesSync(join(attachmentsRoot(), shared), hoursAgo, hoursAgo);

    // Unreferenced when the sweep took its snapshot, and claimed by the time
    // the unlink loop reaches it — a dedupe fetch that landed on these exact
    // bytes through link()'s EEXIST. The age gate cannot see that: the file is
    // genuinely old, it is the REFERENCE that is new.
    respondTo([
      ['SELECT DISTINCT local_path FROM beeper_attachments WHERE local_path IS NOT NULL', { rows: [] }],
      ['WHERE local_path = $1 LIMIT 1', { rows: [{ '?column?': 1 }] }],
    ]);

    const result = await sweepAttachmentOrphans();

    // Otherwise the row that just claimed these bytes reports `stored: true`
    // against a file that is gone.
    expect(result.orphansRemoved).toBe(0);
    expect(existsSync(join(attachmentsRoot(), shared))).toBe(true);
  });

  it('reads a store it cannot list as unreadable rather than empty, and heals no row', async () => {
    // The store root is replaced by a FILE, so every listing of it fails
    // (ENOTDIR) exactly the way an EACCES or a data volume that came back
    // unmounted would. A `catch(() => [])` reads that as "no files on disk" and
    // then clears every mirrored row's claim on its bytes in one pass.
    const root = attachmentsRoot();
    mkdirSync(dirname(root), { recursive: true });
    writeFileSync(root, Buffer.alloc(0));

    const referencedPath = `bb/${'b'.repeat(64)}.png`;
    const healed = [];
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('SELECT DISTINCT local_path FROM beeper_attachments WHERE local_path IS NOT NULL')) {
        return { rows: [{ local_path: referencedPath }] };
      }
      if (flat.includes('WHERE local_path = ANY')) { healed.push(params[0]); return { rowCount: 1 }; }
      return { rows: [] };
    });
    const errors = [];
    const errorSpy = vi.spyOn(console, 'error').mockImplementation((line) => { errors.push(String(line)); });

    const result = await sweepAttachmentOrphans();

    expect(result).toMatchObject({ storeListed: false, healedRows: 0 });
    // Not one `local_path` cleared, and the failure is said out loud rather
    // than read as an empty store.
    expect(healed).toEqual([]);
    expect(errors.some((line) => line.includes('could not be listed'))).toBe(true);
    expect(errors.some((line) => line.includes('no row healed this pass'))).toBe(true);
    errorSpy.mockRestore();
  });

  it('names a failed unlink in the sweep log line rather than swallowing it', async () => {
    const blocked = `ba/${'ba'.repeat(32)}.png`;
    mkdirSync(join(attachmentsRoot(), blocked), { recursive: true });
    const hoursAgo = new Date(Date.now() - 4 * 60 * 60 * 1000);
    utimesSync(join(attachmentsRoot(), blocked), hoursAgo, hoursAgo);
    respondTo([['SELECT DISTINCT local_path FROM beeper_attachments WHERE local_path IS NOT NULL', { rows: [] }]]);
    const logs = [];
    const logSpy = vi.spyOn(console, 'log').mockImplementation((line) => { logs.push(String(line)); });
    const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});

    const result = await sweepAttachmentOrphans();

    // Swallowed, this is a sweep that reports "0 orphan file(s)" and stays
    // silent while the bytes it meant to reclaim are still there.
    expect(result).toMatchObject({ orphansRemoved: 0, failedUnlinks: 1 });
    expect(logs.some((line) => line.includes('1 unlink failure(s)'))).toBe(true);
    logSpy.mockRestore();
    errorSpy.mockRestore();
  });

  it('spares an unreferenced file young enough to be a download still finishing', async () => {
    const fresh = `dd/${'d'.repeat(64)}.png`;
    mkdirSync(join(attachmentsRoot(), 'dd'), { recursive: true });
    writeFileSync(join(attachmentsRoot(), fresh), Buffer.alloc(8));
    respondTo([['SELECT DISTINCT local_path FROM beeper_attachments WHERE local_path IS NOT NULL', { rows: [] }]]);

    const result = await sweepAttachmentOrphans();
    expect(result.orphansRemoved).toBe(0);
    expect(existsSync(join(attachmentsRoot(), fresh))).toBe(true);
  });
});

describe('the backfill census the consent step has to state', () => {
  it('counts unknown-size attachments separately instead of adding them as zero', async () => {
    respondTo([
      ['unique_files', { rows: [{ bytes: '2048', files: 2 }] }],
      ['COUNT(*) FILTER', { rows: [{ pending: 12, pending_bytes: '4096', pending_unknown: 3, over_cap: 1, unavailable: 2, kept: 1, total: 20 }] }],
    ]);
    const summary = await getAttachmentSummary();
    expect(summary).toMatchObject({
      usedBytes: 2048, pendingCount: 12, pendingBytes: 4096, pendingUnknownCount: 3, overCapCount: 1,
    });
    expect(summary.budgetBytes).toBe(5 * 1024 * 1024 * 1024);
  });

  it('counts a failed attachment and carries on through the rest of the run', async () => {
    const pending = [
      { message_id: 'msg-example-1', idx: 0 },
      { message_id: 'msg-example-2', idx: 0 },
    ];
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('ORDER BY created_at ASC')) return { rows: pending };
      if (flat.includes('unique_files')) return { rows: [{ bytes: '0', files: 0 }] };
      if (flat.includes('FROM beeper_attachments WHERE message_id')) {
        return { rows: [attachmentRow({ message_id: 'msg-example-1' })] };
      }
      return { rows: [] };
    });
    vi.mocked(headAsset)
      .mockRejectedValueOnce(unavailableError())
      .mockResolvedValueOnce({ bytes: 4 });
    vi.mocked(fetchAssetStream).mockResolvedValue(streamResponse([new Uint8Array([7, 7, 7, 7])]));

    const result = await backfillAttachments({ limit: 10 });
    expect(result).toMatchObject({ requested: 2, fetched: 1, failed: 1, stoppedForBudget: false });
  });

  it('stops at the disk budget rather than blowing through it', async () => {
    vi.mocked(getSettings).mockResolvedValue({ beeper: { attachmentBudgetGb: 0.000001 } });
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('ORDER BY created_at ASC')) return { rows: [{ message_id: 'msg-example-1', idx: 0 }] };
      if (flat.includes('unique_files')) return { rows: [{ bytes: '999999999', files: 9 }] };
      return { rows: [] };
    });

    const result = await backfillAttachments({});
    expect(result).toMatchObject({ fetched: 0, stoppedForBudget: true });
    expect(vi.mocked(fetchAssetStream)).not.toHaveBeenCalled();
  });

  // PERF-4: `storedBytes()` is a DISTINCT ON scan of the whole table — this
  // pins that a multi-attachment pass costs ONE such scan, not one per row.
  it('computes the byte-count total once per pass, not once per attachment', async () => {
    const pending = [
      { message_id: 'msg-example-1', idx: 0 },
      { message_id: 'msg-example-2', idx: 0 },
      { message_id: 'msg-example-3', idx: 0 },
    ];
    let uniqueFilesCalls = 0;
    vi.mocked(query).mockImplementation(async (sql) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('ORDER BY created_at ASC')) return { rows: pending };
      if (flat.includes('unique_files')) {
        uniqueFilesCalls += 1;
        return { rows: [{ bytes: '0', files: 0 }] };
      }
      if (flat.includes('FROM beeper_attachments WHERE message_id')) return { rows: [attachmentRow()] };
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 4 });
    vi.mocked(fetchAssetStream).mockImplementation(async () => streamResponse([new Uint8Array([1, 2, 3, 4])]));

    const result = await backfillAttachments({ limit: 10 });
    expect(result.fetched).toBe(3);
    expect(uniqueFilesCalls).toBe(1);
  });
});

describe('shapeAttachment', () => {
  it('reads an unknown byte length as unknown, never as over-cap', () => {
    expect(shapeAttachment(attachmentRow({ byte_length: null }))).toMatchObject({
      byteLength: null, overCap: false, stored: false, unavailable: false,
    });
  });

  it('separates "stored" from "unavailable" — a mirrored file whose source aged out is both', () => {
    const shaped = shapeAttachment(attachmentRow({
      local_path: `aa/${'a'.repeat(64)}.png`,
      unavailable_at: '2026-09-01T00:00:00.000Z',
      fetch_error: 'Failed to download asset',
      byte_length: 40 * 1024 * 1024,
    }));
    expect(shaped).toMatchObject({ stored: true, unavailable: true, overCap: true });
  });
});


describe('attachment snapshot publication', () => {
  const deferred = () => Promise.withResolvers();
  function mirror(beforeCommit = async () => {}) {
    let row = attachmentRow();
    vi.mocked(query).mockImplementation(async (sql, params) => {
      const flat = String(sql).replace(/\s+/g, ' ');
      if (flat.includes('FROM beeper_attachments WHERE message_id')) return { rows: [row] };
      if (flat.includes('SET local_path = $3')) {
        await beforeCommit(params[2]);
        row = { ...row, local_path: params[2], byte_length: params[3] };
      }
      if (flat.includes('SELECT DISTINCT local_path')) return { rows: row.local_path ? [{ local_path: row.local_path }] : [] };
      if (flat.includes('SELECT 1 FROM beeper_attachments')) return { rows: row.local_path ? [{}] : [] };
      return { rows: [] };
    });
    vi.mocked(headAsset).mockResolvedValue({ bytes: 4 });
    vi.mocked(fetchAssetStream).mockImplementation(async () => streamResponse([new Uint8Array([1, 2, 3, 4])]));
    return { row: () => row, forget: () => { row = { ...row, local_path: null }; } };
  }

  it.each([false, true])('keeps a new path outside an active file-copy cut (bypass=%s)', async bypass => {
    const { acquireBackupSnapshotCut } = await import('../lib/backupSnapshotBoundary.js');
    const table = mirror();
    const arrived = deferred();
    publicationSeam.reached = arrived.resolve;
    publicationSeam.bypass = bypass;
    const release = await acquireBackupSnapshotCut();
    const copiedFiles = storedFiles();
    const work = ensureAttachmentBytes('msg-example-1', 0);
    try {
      await arrived.promise;
      if (bypass) await work;
      expect(table.row().local_path != null).toBe(bypass);
      // Negative control: a post-copy database dump names bytes not copied.
      expect(table.row().local_path != null && !copiedFiles.includes(table.row().local_path)).toBe(bypass);
    } finally { release(); await work; }
    expect(readFileSync(join(attachmentsRoot(), table.row().local_path))).toEqual(Buffer.from([1, 2, 3, 4]));
  });

  it('drains an installed file through its row commit, including row-write failure', async () => {
    const { acquireBackupSnapshotCut } = await import('../lib/backupSnapshotBoundary.js');
    const reached = deferred(); const finish = deferred();
    const table = mirror(async () => { reached.resolve(); await finish.promise; throw new Error('row unavailable'); });
    const work = ensureAttachmentBytes('msg-example-1', 0);
    const rejected = expect(work).rejects.toThrow('row unavailable');
    await reached.promise;
    expect(storedFiles()).toHaveLength(1);
    let cutReady = false;
    const cut = acquireBackupSnapshotCut().then(release => { cutReady = true; return release; });
    try {
      await new Promise(resolve => setImmediate(resolve));
      expect(cutReady).toBe(false);
    } finally { finish.resolve(); }
    await rejected;
    const release = await cut;
    try {
      expect(table.row().local_path).toBeNull();
      // Unclaimed immutable bytes are safe; cleanup must not remove deduped data.
      expect(storedFiles()).toHaveLength(1);
    } finally { release(); }
  });

  it('serializes orphan deletion behind a deduplicated file claim', async () => {
    let block = false;
    const reached = deferred(); const finish = deferred();
    const table = mirror(async () => { if (block) { reached.resolve(); await finish.promise; } });
    const first = await ensureAttachmentBytes('msg-example-1', 0);
    table.forget(); block = true;
    const old = new Date(Date.now() - 2 * 60 * 60 * 1000);
    utimesSync(first.filePath, old, old);
    const claim = ensureAttachmentBytes('msg-example-1', 0);
    await reached.promise;
    const deleting = deferred();
    publicationSeam.reached = deleting.resolve;
    const sweep = sweepAttachmentOrphans();
    try {
      await deleting.promise;
      expect(existsSync(first.filePath)).toBe(true);
    } finally { finish.resolve(); }
    await claim;
    expect(await sweep).toMatchObject({ orphansRemoved: 0 });
    expect(table.row().local_path).not.toBeNull();
    expect(existsSync(first.filePath)).toBe(true);
  });
});