Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
37.5 KB
Raw Blame
import { assertManagedFilesystemWrite } from '../core/persistence/filesystem-guard.ts';
import { readFileSync, readdirSync, statSync, lstatSync, existsSync, writeFileSync, unlinkSync, mkdirSync, copyFileSync } from 'fs';
import { join, relative, extname, basename, dirname, resolve } from 'path';
import { createHash } from 'crypto';
import type { BrainEngine } from '../core/engine.ts';
import { hashStorageFile, uploadUnchangedFile, type StorageBackend, type StorageConfig } from '../core/storage.ts';
import { sqlQueryForEngine, executeRawJsonb, FILES_METADATA_MERGE_SQL } from '../core/sql-query.ts';
import { humanSize } from '../core/file-resolver.ts';
import { createProgress } from '../core/progress.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';

/** Size threshold: files >= 100 MB use TUS resumable upload */
const SIZE_THRESHOLD = 100 * 1024 * 1024;

interface FileRecord {
  id: number;
  page_slug: string | null;
  filename: string;
  storage_path: string;
  mime_type: string | null;
  size_bytes: number | bigint | string | null;
  content_hash: string;
  metadata: Record<string, unknown>;
  created_at: string;
}

const MIME_TYPES: Record<string, string> = {
  '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.png': 'image/png',
  '.gif': 'image/gif', '.webp': 'image/webp', '.svg': 'image/svg+xml',
  '.pdf': 'application/pdf', '.mp4': 'video/mp4', '.m4a': 'audio/mp4',
  '.mp3': 'audio/mpeg', '.wav': 'audio/wav', '.heic': 'image/heic',
  '.ogg': 'audio/ogg', '.flac': 'audio/flac', '.mpga': 'audio/mpeg',
  '.webm': 'video/webm', '.mpeg': 'video/mpeg',
  '.tiff': 'image/tiff', '.tif': 'image/tiff', '.dng': 'image/x-adobe-dng',
  '.doc': 'application/msword', '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
  '.xls': 'application/vnd.ms-excel', '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
};

export function getMimeType(filePath: string): string | null {
  const ext = extname(filePath).toLowerCase();
  return MIME_TYPES[ext] || null;
}

function fileHash(filePath: string): string {
  return hashStorageFile(filePath);
}

export function formatFileSizeKb(rawSizeBytes: number | bigint | string | null): string {
  if (rawSizeBytes == null) return '?';
  const sizeBytes = Number(rawSizeBytes);
  return Number.isFinite(sizeBytes) && sizeBytes >= 0
    ? `${Math.round(sizeBytes / 1024)}KB`
    : '?';
}

/**
 * The message every storage-dependent `files` subcommand prints when there is
 * no backend. Exported so tests assert the contract without spawning a CLI.
 */
export function noStorageBackendMessage(op: string): string {
  return (
    `gbrain files ${op}: no storage backend configured — refusing to continue.\n` +
    `  The files table records metadata only (there is no blob column), so without a\n` +
    `  backend the bytes go nowhere while the DB claims they are stored.\n` +
    `  Fix: configure storage (see gbrain init storage settings), or keep the binary\n` +
    `  outside the brain and capture its extracted text instead.`
  );
}

/**
 * Single precondition for every storage-dependent `files` subcommand (#4022).
 *
 * Class fix (see `noStorageBackendMessage`): `upload`, `sync`, and `redirect`
 * each tested storage permissively (`if (config?.storage)`) and then carried on
 * when the answer was "no" — inserting rows, printing "uploaded", and in
 * `redirect` unlinking local originals whose bytes had never left the machine.
 * Storage-dependent work must refuse up front rather than half-succeed, so this
 * exits BEFORE any DB write or local mutation.
 */
async function requireStorageBackend(
  op: string,
): Promise<{ storage: StorageBackend; storageConfig: StorageConfig }> {
  const { loadConfig } = await import('../core/config.ts');
  const config = loadConfig();
  if (!config?.storage) {
    console.error(noStorageBackendMessage(op));
    process.exit(1);
  }
  const { createStorage } = await import('../core/storage.ts');
  const storageConfig = config.storage as StorageConfig;
  return { storage: await createStorage(storageConfig), storageConfig };
}

export async function runFiles(engine: BrainEngine, args: string[]) {
  const subcommand = args[0];

  switch (subcommand) {
    case 'list':
      await listFiles(engine, args[1]);
      break;
    case 'upload':
      await uploadFile(engine, args.slice(1));
      break;
    case 'sync':
      await syncFiles(engine, args[1]);
      break;
    case 'verify':
      await verifyFiles(engine);
      break;
    case 'mirror':
      await mirrorFiles(args.slice(1));
      break;
    case 'unmirror':
      await unmirrorFiles(args.slice(1));
      break;
    case 'redirect':
      await redirectFiles(args.slice(1));
      break;
    case 'restore':
      await restoreFiles(args.slice(1));
      break;
    case 'clean':
      await cleanFiles(args.slice(1));
      break;
    case 'upload-raw':
      await uploadRaw(engine, args.slice(1));
      break;
    case 'signed-url':
      await signedUrl(args.slice(1));
      break;
    case 'status':
      await filesStatus(args.slice(1));
      break;
    default:
      console.error(`Usage: gbrain files <command> [args]`);
      console.error(`  list [slug]               List files for a page (or all)`);
      console.error(`  upload <file> --page <slug>  Upload file linked to page`);
      console.error(`  upload-raw <file> --page <slug> [--type <type>]  Smart upload with .redirect.yaml pointer`);
      console.error(`  signed-url <path>         Generate signed URL for stored file`);
      console.error(`  sync <dir>                Upload directory to storage`);
      console.error(`  verify                    Verify all uploads match local`);
      console.error(`  mirror <dir> [--dry-run]  Mirror files to cloud storage`);
      console.error(`  unmirror <dir>            Remove mirror marker (files stay in storage)`);
      console.error(`  redirect <dir> [--dry-run]  Replace files with .redirect.yaml pointers`);
      console.error(`  restore <dir>             Download from storage, recreate local files`);
      console.error(`  clean <dir> [--yes]       Delete redirect pointers (irreversible)`);
      console.error(`  status                    Show migration status of directories`);
      process.exit(1);
  }
}

async function listFiles(engine: BrainEngine, slug?: string) {
  const sql = sqlQueryForEngine(engine);
  let rows;
  if (slug) {
    rows = await sql`SELECT * FROM files WHERE page_slug = ${slug} ORDER BY filename LIMIT 100`;
  } else {
    rows = await sql`SELECT * FROM files ORDER BY page_slug, filename LIMIT 100`;
  }

  if (rows.length === 0) {
    console.log(slug ? `No files for page: ${slug}` : 'No files stored.');
    return;
  }

  console.log(`${rows.length} file(s):`);
  for (const row of rows) {
    const size = formatFileSizeKb(row.size_bytes as FileRecord['size_bytes']);
    console.log(`  ${row.page_slug || '(unlinked)'} / ${row.filename}  [${size}, ${row.mime_type || '?'}]`);
  }
}

async function uploadFile(engine: BrainEngine, args: string[]) {
  const filePath = args.find(a => !a.startsWith('--'));
  const pageSlug = args.find((a, i) => args[i - 1] === '--page') || null;

  if (!filePath || !existsSync(filePath)) {
    console.error('Usage: gbrain files upload <file> --page <slug>');
    process.exit(1);
  }

  // Precondition first: a backend-less upload can only produce a phantom row
  // (metadata for bytes that were never stored), so refuse before touching the
  // DB or reporting anything as uploaded.
  const { storage, storageConfig } = await requireStorageBackend('upload');

  const stat = statSync(filePath);
  const hash = fileHash(filePath);
  const filename = basename(filePath);
  const storagePath = pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`;
  const mimeType = getMimeType(filePath);

  const sql = sqlQueryForEngine(engine);

  // Check for existing file by hash — but only trust the row when the
  // BACKEND really holds the object (#4302); vanished bytes must re-upload.
  const existing = await sql`SELECT id FROM files WHERE content_hash = ${hash} AND storage_path = ${storagePath}`;
  if (existing.length > 0 && (await storage.exists(storagePath).catch(() => false))) {
    console.log(`File already uploaded (hash match): ${storagePath}`);
    return;
  }

  const method = stat.size >= SIZE_THRESHOLD ? 'TUS resumable' : 'standard';
  console.log(`Uploading ${humanSize(stat.size)} via ${method}...`);
  await uploadUnchangedFile(storage, storagePath, filePath, hash, mimeType || undefined);

  // files.metadata is JSONB — bind a real object via executeRawJsonb instead
  // of casting a string into ::jsonb (the #2339 double-encode class).
  // #4910: stamp the lane so readers (doctor image_assets, files verify)
  // know storage_path is a bucket key; the upsert merges metadata so a
  // legacy `{}` row heals on its next content change.
  await executeRawJsonb(
    engine,
    `INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
     VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
     ON CONFLICT (storage_path) DO UPDATE SET
       content_hash = EXCLUDED.content_hash,
       size_bytes = EXCLUDED.size_bytes,
       mime_type = EXCLUDED.mime_type,
       ${FILES_METADATA_MERGE_SQL}`,
    [pageSlug, filename, storagePath, mimeType, stat.size, hash],
    [{ storage: storageConfig.backend }],
  );

  console.log(`Uploaded: ${storagePath} (${humanSize(stat.size)})`);
}

/**
 * Smart upload with size routing and .redirect.yaml pointer creation.
 *
 * Size routing:
 *   < 100 MB text/PDF  → stays in git (brain repo), no cloud upload
 *   >= 100 MB OR media  → upload to cloud storage, create .redirect.yaml pointer
 *
 * The .redirect.yaml pointer stays in the brain repo so git tracks what was stored.
 * On a managed canonical worktree the pointer write is refused before the upload (#7346).
 */
async function uploadRaw(engine: BrainEngine, args: string[]) {
  const filePath = args.find(a => !a.startsWith('--'));
  const pageSlug = args.find((a, i) => args[i - 1] === '--page') || null;
  const fileType = args.find((a, i) => args[i - 1] === '--type') || null;
  const noPointer = args.includes('--no-pointer');

  if (!filePath || !existsSync(filePath)) {
    console.error('Usage: gbrain files upload-raw <file> --page <slug> [--type <type>] [--no-pointer]');
    process.exit(1);
  }

  const stat = statSync(filePath);
  const filename = basename(filePath);
  // A file argument that IS `.` or `..` makes basename() return that
  // literal string back rather than a real leaf filename (a trailing
  // separator, e.g. `foo/`, is stripped by basename() to `foo` — not
  // affected). The git-storage branch below joins this value onto its
  // sidecar dest dir (`destDir/${filename}`) — a `..` segment there walks
  // the join back up OUT of the intended `.raw/<page>/` dir before the
  // copy. Reject early with a clear error instead of letting it silently
  // resolve to the parent dir and fail deep inside copyFileSync with a
  // confusing OS-level error.
  if (filename === '.' || filename === '..') {
    console.error(`files upload-raw: "${filePath}" does not name a real file (resolves to "${filename}").`);
    process.exit(1);
  }
  const mimeType = getMimeType(filePath);
  const isMedia = mimeType?.startsWith('video/') || mimeType?.startsWith('audio/') || mimeType?.startsWith('image/');
  const needsCloud = stat.size >= SIZE_THRESHOLD || isMedia;

  if (!needsCloud) {
    // #2297: small text/PDF files "stay in git" — which used to mean the
    // command did NOTHING (no repo copy, no files row) while printing
    // success:true. Actually bank the file: copy it into the page's .raw/
    // sidecar dir inside the brain repo and record a files row so
    // `gbrain files list` / `verify` can see it.
    if (!pageSlug) {
      console.error('files upload-raw: --page <slug> is required for git-storage (small text/PDF) files.');
      process.exit(1);
    }
    const { resolveSourceId } = await import('../core/source-resolver.ts');
    const { resolvePageWriteTarget } = await import('../core/write-through.ts');
    const sourceArg = args.find((a, i) => args[i - 1] === '--source') || null;
    const sourceId = await resolveSourceId(engine, sourceArg);
    const target = await resolvePageWriteTarget(engine, pageSlug, sourceId);
    if (!target.ok) {
      console.error(
        `files upload-raw: cannot resolve a brain-repo destination for page "${pageSlug}" ` +
        `(${target.skipped}). Configure sync.repo_path (or the source's local_path), ` +
        `or use \`gbrain files upload\` with a cloud storage backend.`,
      );
      process.exit(1);
    }
    // <pageDir>/.raw/<page-name>/<basename> — sidecar layout next to the
    // page's canonical markdown artifact.
    const pageDir = dirname(target.filePath);
    const pageName = basename(target.filePath).replace(/\.md$/i, '');
    // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- pageDir/pageName derive from resolvePageWriteTarget's brain-repo target for the operator's own --page slug; upload-raw is a trusted-local CLI lane (runFiles is wired only from cli.ts, remote:false — the MCP file_upload op is a separate localOnly handler)
    const destDir = join(pageDir, '.raw', pageName);
    // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- filename is basename() of the operator's own CLI file arg (basename yields a single separator-free segment), joined under the brain-repo sidecar dir above
    const dest = join(destDir, filename);
    // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- identity comparison only (skip self-copy when source already IS the dest); no fs path is derived from this expression
    if (resolve(dest) !== resolve(filePath)) {
      // #5963: refuse on a managed root before mkdir, so a refused upload leaves no empty .raw/ directory.
      const refusal = (await import('../core/raw-file-upload.ts')).managedRawUploadRefusal(dest, filePath, pageSlug);
      if (refusal) throw refusal;
      mkdirSync(destDir, { recursive: true });
      assertManagedFilesystemWrite(dest);
      copyFileSync(filePath, dest);
    }
    const hash = fileHash(filePath);
    const storagePath = relative(target.writeRoot, dest);
    await executeRawJsonb(
      engine,
      `INSERT INTO files (source_id, page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
       VALUES ($1, $2, $3, $4, $5, $6, $7, $8::jsonb)
       ON CONFLICT (storage_path) DO UPDATE SET
         content_hash = EXCLUDED.content_hash,
         size_bytes = EXCLUDED.size_bytes,
         mime_type = EXCLUDED.mime_type,
         ${FILES_METADATA_MERGE_SQL}`,
      [sourceId, pageSlug, filename, storagePath, mimeType, stat.size, 'sha256:' + hash],
      [{ storage: 'git', ...(fileType ? { type: fileType } : {}) }],
    );
    console.log(JSON.stringify({
      success: true,
      storage: 'git',
      path: dest,
      storagePath,
      size: stat.size,
      size_human: humanSize(stat.size),
      hash: `sha256:${hash}`,
    }));
    return;
  }

  // Upload to cloud storage
  const { loadConfig } = await import('../core/config.ts');
  const config = loadConfig();
  if (!config?.storage) {
    console.error('No storage backend configured. Run gbrain init with storage settings.');
    console.error('Or use gbrain files upload for manual uploads.');
    process.exit(1);
  }

  if (!noPointer && pageSlug) {
    const refusal = (await import('../core/raw-file-upload.ts')).managedPointerRefusal(filePath, pageSlug, fileType);
    if (refusal) throw refusal;
  }

  const { createStorage } = await import('../core/storage.ts');
  const storage = await createStorage(config.storage as any);
  const hash = fileHash(filePath);
  const storagePath = pageSlug ? `${pageSlug}/${filename}` : `unsorted/${hash.slice(0, 8)}-${filename}`;
  const bucket = (config.storage as any).bucket || 'brain-files';
  const backend = (config.storage as StorageConfig).backend;
  const reference = `${backend}://${bucket}/${storagePath}`;

  const method = stat.size >= SIZE_THRESHOLD ? 'TUS resumable' : 'standard';
  console.error(`Uploading ${humanSize(stat.size)} via ${method}...`);
  await uploadUnchangedFile(storage, storagePath, filePath, hash, mimeType || undefined);

  // Create .redirect.yaml pointer in the brain repo
  let pointerPath: string | null = null;
  if (!noPointer && pageSlug) {
    const { stringify } = await import('../core/yaml-lite.ts');
    const pointer = stringify({
      target: reference,
      bucket,
      storage_path: storagePath,
      size: stat.size,
      size_human: humanSize(stat.size),
      hash: `sha256:${hash}`,
      mime: mimeType || 'application/octet-stream',
      uploaded: new Date().toISOString(),
      ...(fileType ? { type: fileType } : {}),
    });
    // Write pointer next to the original file
    pointerPath = filePath + '.redirect.yaml';
    assertManagedFilesystemWrite(pointerPath);
    writeFileSync(pointerPath, pointer);
    console.error(`Pointer written: ${pointerPath}`);
  }

  // Record in DB. files.metadata is JSONB — pass the object via
  // executeRawJsonb with an explicit ::jsonb cast so post-v0.31 reads see
  // an actual object, not a JSON-encoded string (D1 wave).
  await executeRawJsonb(
    engine,
    `INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
     VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
     ON CONFLICT (storage_path) DO UPDATE SET
       content_hash = EXCLUDED.content_hash,
       size_bytes = EXCLUDED.size_bytes,
       mime_type = EXCLUDED.mime_type,
       ${FILES_METADATA_MERGE_SQL}`,
    [pageSlug, filename, storagePath, mimeType, stat.size, 'sha256:' + hash],
    [{ storage: backend, ...(fileType ? { type: fileType } : {}), upload_method: method }],
  );

  // Output JSON for scripting
  console.log(JSON.stringify({
    success: true,
    storage: backend,
    storagePath,
    bucket,
    reference,
    pointerPath,
    size: stat.size,
    size_human: humanSize(stat.size),
    hash: `sha256:${hash}`,
    upload_method: method,
  }));
}

/** Generate a signed URL for a stored file */
async function signedUrl(args: string[]) {
  const storagePath = args.find(a => !a.startsWith('--'));
  if (!storagePath) {
    console.error('Usage: gbrain files signed-url <storage-path>');
    process.exit(1);
  }

  const { loadConfig } = await import('../core/config.ts');
  const config = loadConfig();
  if (!config?.storage) {
    console.error('No storage backend configured.');
    process.exit(1);
  }

  const { createStorage } = await import('../core/storage.ts');
  const storage = await createStorage(config.storage as any);
  const url = await storage.getUrl(storagePath);
  console.log(url);
}

async function syncFiles(engine: BrainEngine, dir?: string) {
  if (!dir || !existsSync(dir)) {
    console.error('Usage: gbrain files sync <directory>');
    process.exit(1);
  }

  // Pre-fix this command inserted a `files` row per file and reported them as
  // "uploaded" without ever calling the storage backend — every row it produced
  // was a phantom, even on a brain WITH storage configured.
  const { storage, storageConfig } = await requireStorageBackend('sync');

  const files = collectFiles(dir);
  console.log(`Found ${files.length} files to sync`);

  let uploaded = 0;
  let skipped = 0;

  const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
  progress.start('files.sync', files.length);

  for (let i = 0; i < files.length; i++) {
    const filePath = files[i];
    const relativePath = relative(dir, filePath);

    progress.tick(1);

    const stat = statSync(filePath);
    const hash = fileHash(filePath);
    const filename = basename(filePath);
    const storagePath = relativePath.replace(/\\/g, '/');
    const mimeType = getMimeType(filePath);

    const sql = sqlQueryForEngine(engine);
    const existing = await sql`SELECT id FROM files WHERE content_hash = ${hash} AND storage_path = ${storagePath}`;
    if (existing.length > 0) {
      skipped++;
      continue;
    }

    // Infer page slug from directory structure
    const pathParts = relativePath.split('/');
    const pageSlug = pathParts.length > 1 ? pathParts.slice(0, -1).join('/') : null;

    // Actually put the bytes in storage before recording them as stored.
    await uploadUnchangedFile(storage, storagePath, filePath, hash, mimeType || undefined);

    // files.metadata is JSONB — bind a real object via executeRawJsonb instead
    // of casting a string into ::jsonb (the #2339 double-encode class).
    await executeRawJsonb(
      engine,
      `INSERT INTO files (page_slug, filename, storage_path, mime_type, size_bytes, content_hash, metadata)
       VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb)
       ON CONFLICT (storage_path) DO UPDATE SET
         content_hash = EXCLUDED.content_hash,
         size_bytes = EXCLUDED.size_bytes,
         mime_type = EXCLUDED.mime_type,
         ${FILES_METADATA_MERGE_SQL}`,
      [pageSlug, filename, storagePath, mimeType, stat.size, hash],
      [{ storage: storageConfig.backend }],
    );

    uploaded++;
  }

  progress.finish();
  // Stdout summary preserved for scripts/tests that grep for it.
  console.log(`Files sync complete: ${uploaded} uploaded, ${skipped} skipped (unchanged)`);
}

async function verifyFiles(engine: BrainEngine) {
  const sql = sqlQueryForEngine(engine);
  const rows = await sql`SELECT * FROM files ORDER BY storage_path LIMIT 1000`;

  if (rows.length === 0) {
    console.log('No files to verify.');
    return;
  }

  // Pre-fix this loop asked only "does the ROW carry a hash and a path" — true
  // for any row an INSERT produced — and then printed a hardcoded
  // "0 mismatches, 0 missing". `missing` was declared and never incremented.
  // So the one command whose job is catching un-stored files reported phantom
  // rows as "verified". The check that matters is byte existence at
  // storage_path, which requires the backend (#4022) — except git-lane rows
  // (#4302), which verify against the brain repo on disk.
  let verified = 0;
  let mismatches = 0;
  let missing = 0;
  // #4022: cloud rows with no configured (or constructible) backend are
  // UNVERIFIABLE, never "verified" — almost certainly phantoms left by a
  // backend-less upload/sync. Counted per row so a git-lane-only brain
  // still passes without a backend.
  let unverifiable = 0;

  // #4302: verify against the actual bytes, not just DB row shape. Cloud rows
  // are probed via storage.exists (+ hash-checked via download for small
  // objects); git-storage rows (metadata.storage === 'git', the upload-raw
  // small-file lane) are checked against the brain repo on disk.
  const { loadConfig } = await import('../core/config.ts');
  const config = loadConfig();
  const storage = config?.storage
    ? await (await import('../core/storage.ts')).createStorage(config.storage as any).catch(() => null)
    : null;
  const repoPath = await engine.getConfig('sync.repo_path').catch(() => null);
  // Git-lane root resolution mirrors upload-raw's resolvePageWriteTarget:
  // storage_path was banked relative to the row's OWNING source's local_path
  // when that source has its own working tree, with sync.repo_path only as
  // the fallback. Joining every row against sync.repo_path falsely reported
  // MISSING (or hash-checked the wrong file) for separate-tree sources.
  const sourceLocalPathCache = new Map<string, string | null>();
  const gitRootFor = async (sourceId: unknown): Promise<string | null> => {
    const sid = typeof sourceId === 'string' && sourceId ? sourceId : null;
    if (!sid) return repoPath ? String(repoPath) : null;
    if (!sourceLocalPathCache.has(sid)) {
      let localPath: string | null = null;
      try {
        const srcRows = await sql`SELECT local_path FROM sources WHERE id = ${sid}`;
        localPath = srcRows[0]?.local_path ? String(srcRows[0].local_path) : null;
      } catch { /* sources table unavailable — fall back to sync.repo_path */ }
      sourceLocalPathCache.set(sid, localPath);
    }
    return sourceLocalPathCache.get(sid) ?? (repoPath ? String(repoPath) : null);
  };
  const HASH_CHECK_MAX_BYTES = 10 * 1024 * 1024;
  const normalizeHash = (h: string) => h.replace(/^sha256:/, '');

  for (const row of rows) {
    if (!row.content_hash || !row.storage_path) {
      mismatches++;
      console.error(`  MISMATCH: ${row.storage_path} (missing hash or path)`);
      continue;
    }
    const storagePath = String(row.storage_path);
    const meta = (typeof row.metadata === 'string' ? JSON.parse(row.metadata) : row.metadata) as Record<string, unknown> | null;
    if (meta?.storage === 'git') {
      const gitRoot = await gitRootFor(row.source_id);
      // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal -- gitRoot is the operator-written sources.local_path / sync.repo_path config; storage_path rows are banked exclusively by trusted-local writers (upload-raw's relative(writeRoot, dest) on the CLI, plus the localOnly file_upload op) — files verify is itself a CLI-only read that hash-compares and reports, never serves content
      const local = gitRoot ? join(gitRoot, storagePath) : null;
      if (!local || !existsSync(local)) {
        missing++;
        console.error(`  MISSING: ${row.storage_path} (git-storage file not in brain repo)`);
      } else if (normalizeHash(fileHash(local)) !== normalizeHash(String(row.content_hash))) {
        mismatches++;
        console.error(`  MISMATCH: ${row.storage_path} (repo file hash differs from DB record)`);
      } else {
        verified++;
      }
      continue;
    }
    if (storage) {
      const present = await storage.exists(storagePath).catch(() => false);
      if (!present) {
        missing++;
        console.error(`  MISSING: ${row.storage_path} (not in storage backend)`);
        continue;
      }
      const size = row.size_bytes == null ? null : Number(row.size_bytes);
      if (size !== null && size <= HASH_CHECK_MAX_BYTES) {
        try {
          const bytes = await storage.download(storagePath);
          const actual = createHash('sha256').update(bytes).digest('hex');
          if (actual !== normalizeHash(String(row.content_hash))) {
            mismatches++;
            console.error(`  MISMATCH: ${row.storage_path} (backend hash differs from DB record)`);
            continue;
          }
        } catch { /* download hiccup — exists() already vouched; count verified */ }
      }
      verified++;
    } else {
      // #4022: no backend — byte existence cannot be checked; report, never vouch.
      unverifiable++;
      console.error(`  UNVERIFIABLE: ${row.storage_path} (no storage backend configured)`);
    }
  }
  if (unverifiable > 0) {
    console.error(
      `gbrain files verify: ${unverifiable} cloud file row(s) recorded, but no storage backend is configured.\n` +
      `  Byte existence cannot be checked and these rows cannot be retrieved, so they are\n` +
      `  reported as UNVERIFIABLE rather than "verified" — almost certainly phantoms left by\n` +
      `  a backend-less upload/sync. Configure storage, or delete the rows.`,
    );
  }

  // Always report the real counts — never a hardcoded pair.
  console.log(`${verified} files verified, ${mismatches} mismatches, ${missing} missing, ${unverifiable} unverifiable`);
  if (mismatches > 0 || missing > 0 || unverifiable > 0) {
    console.error(`VERIFY FAILED: ${mismatches} mismatches, ${missing} missing, ${unverifiable} unverifiable.`);
    console.error(`Run: gbrain files sync --retry-failed`);
    process.exit(1);
  }
}

// ─────────────────────────────────────────────────────────────────
// File Migration Commands (mirror → redirect → clean lifecycle)
// ─────────────────────────────────────────────────────────────────

async function mirrorFiles(args: string[]) {
  const dir = args.find(a => !a.startsWith('--'));
  const dryRun = args.includes('--dry-run');
  if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files mirror <dir> [--dry-run]'); process.exit(1); }

  const { stringify } = await import('../core/yaml-lite.ts');
  const { storage, storageConfig } = await requireStorageBackend('mirror');
  const files = collectFiles(dir);
  console.log(`Found ${files.length} files to mirror`);

  if (dryRun) {
    for (const f of files) { console.log(`  Would upload: ${relative(dir, f)}`); }
    console.log(`\nDry run: ${files.length} files would be uploaded.`);
    return;
  }

  let uploaded = 0;
  for (const filePath of files) {
    const relPath = relative(dir, filePath);
    const mime = getMimeType(filePath);
    await storage.uploadFile(relPath, filePath, mime || undefined);
    uploaded++;
  }

  // Write .supabase marker
  const marker = stringify({
    synced_at: new Date().toISOString(),
    bucket: storageConfig?.bucket || 'brain-files',
    prefix: basename(dir) + '/',
    file_count: uploaded,
  });
  assertManagedFilesystemWrite(dir);
  writeFileSync(join(dir, '.supabase'), marker);

  console.log(`Mirrored ${uploaded} files. Marker written to ${dir}/.supabase`);
}

async function unmirrorFiles(args: string[]) {
  const dir = args.find(a => !a.startsWith('--'));
  if (!dir) { console.error('Usage: gbrain files unmirror <dir>'); process.exit(1); }

  const markerPath = join(dir, '.supabase');
  if (existsSync(markerPath)) {
    assertManagedFilesystemWrite(markerPath);
    unlinkSync(markerPath);
    console.log(`Removed mirror marker from ${dir}. Files remain in storage.`);
  } else {
    console.log(`No mirror marker found in ${dir}. Nothing to do.`);
  }
}

async function redirectFiles(args: string[]) {
  const dir = args.find(a => !a.startsWith('--'));
  const dryRun = args.includes('--dry-run');
  if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files redirect <dir> [--dry-run]'); process.exit(1); }

  const markerPath = join(dir, '.supabase');
  if (!existsSync(markerPath)) {
    console.error('Directory must be mirrored first. Run: gbrain files mirror <dir>');
    process.exit(1);
  }

  const { parse: parseYaml, stringify } = await import('../core/yaml-lite.ts');
  const marker = parseYaml(readFileSync(markerPath, 'utf-8'));
  const files = collectFiles(dir);

  if (dryRun) {
    for (const f of files) { console.log(`  Would redirect: ${relative(dir, f)}`); }
    console.log(`\nDry run: ${files.length} files would be redirected.`);
    return;
  }

  // Verify remote files exist before deleting locals.
  //
  // This is the destructive path: it unlinks the local original and leaves a
  // `supabase://` pointer behind. Pre-fix, `storage` was only built when a
  // backend happened to be configured (`if (config?.storage)`) and the
  // existence check was correspondingly conditional (`if (storage)`) — so with
  // no backend the guard was skipped entirely and the loop deleted originals
  // while writing pointers to bytes that had never been uploaded. Data loss,
  // not a phantom row. The backend is now mandatory here.
  const { storage } = await requireStorageBackend('redirect');

  let redirected = 0;
  let skippedMissing = 0;
  for (const filePath of files) {
    const relPath = relative(dir, filePath);
    const hash = fileHash(filePath);

    // Unconditional: never unlink a local original we have not confirmed
    // exists remotely.
    const remoteExists = await storage.exists(relPath);
    if (!remoteExists) {
      console.error(`  Skipping ${relPath}: not found in remote storage (would lose data)`);
      skippedMissing++;
      continue;
    }

    const stat = statSync(filePath);
    const mimeType = getMimeType(filePath);
    const bucket = marker.bucket || 'brain-files';
    const pointer = stringify({
      target: `supabase://${bucket}/${relPath}`,
      bucket,
      storage_path: relPath,
      size: stat.size,
      size_human: humanSize(stat.size),
      hash: `sha256:${hash}`,
      mime: mimeType || 'application/octet-stream',
      uploaded: new Date().toISOString(),
    });
    assertManagedFilesystemWrite(filePath);
    writeFileSync(filePath + '.redirect.yaml', pointer);
    unlinkSync(filePath);
    redirected++;
  }

  console.log(`Redirected ${redirected} files. Originals removed, breadcrumbs created.`);
  if (skippedMissing > 0) {
    console.log(`Skipped ${skippedMissing} files (not found in remote storage — run 'gbrain files mirror' first).`);
  }
  console.log('To undo: gbrain files restore <dir>');
}

async function restoreFiles(args: string[]) {
  const dir = args.find(a => !a.startsWith('--'));
  if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files restore <dir>'); process.exit(1); }

  const { createStorage } = await import('../core/storage.ts');
  const { loadConfig } = await import('../core/config.ts');
  const { parse: parseYaml } = await import('../core/yaml-lite.ts');
  const config = loadConfig();
  if (!config?.storage) { console.error('No storage backend configured.'); process.exit(1); }

  const storage = await createStorage(config.storage as any);
  const redirectFiles: string[] = [];

  function findRedirects(d: string) {
    for (const entry of readdirSync(d)) {
      if (entry.startsWith('.')) continue;
      const full = join(d, entry);
      let stat;
      try {
        stat = lstatSync(full);
      } catch {
        continue; // Broken symlink or permission error
      }
      if (stat.isSymbolicLink()) continue;
      if (stat.isDirectory()) findRedirects(full);
      else if (entry.endsWith('.redirect.yaml') || entry.endsWith('.redirect')) redirectFiles.push(full);
    }
  }
  findRedirects(dir);

  let restored = 0;
  let failed = 0;
  for (const redirectPath of redirectFiles) {
    const info = parseYaml(readFileSync(redirectPath, 'utf-8'));
    const originalPath = redirectPath.replace(/\.redirect(\.yaml)?$/, '');
    try {
      const storagePath = info.storage_path || info.path; // v0.9 or legacy format
      const data = await storage.download(storagePath);
      assertManagedFilesystemWrite(originalPath);
      writeFileSync(originalPath, data);
      unlinkSync(redirectPath);
      restored++;
    } catch (e: unknown) {
      const msg = e instanceof Error ? e.message : String(e);
      console.error(`  Failed to restore ${info.path}: ${msg}`);
      failed++;
    }
  }

  console.log(`Restored ${restored} files. ${failed > 0 ? `${failed} failed.` : ''}`);
}

async function cleanFiles(args: string[]) {
  const dir = args.find(a => !a.startsWith('--'));
  const confirmed = args.includes('--yes');
  if (!dir || !existsSync(dir)) { console.error('Usage: gbrain files clean <dir> [--yes]'); process.exit(1); }

  if (!confirmed) {
    console.error('WARNING: This permanently removes redirect pointers.');
    console.error('After this, files are only accessible from cloud storage.');
    console.error('Git history still has the originals if you need them.');
    console.error('Run with --yes to confirm.');
    process.exit(1);
  }

  let cleaned = 0;
  function findAndClean(d: string) {
    for (const entry of readdirSync(d)) {
      if (entry.startsWith('.')) continue;
      const full = join(d, entry);
      let stat;
      try {
        stat = lstatSync(full);
      } catch {
        continue; // Broken symlink or permission error
      }
      if (stat.isSymbolicLink()) continue;
      if (stat.isDirectory()) findAndClean(full);
      else if (entry.endsWith('.redirect.yaml') || entry.endsWith('.redirect')) { assertManagedFilesystemWrite(full); unlinkSync(full); cleaned++; }
    }
  }
  findAndClean(dir);

  console.log(`Cleaned ${cleaned} redirect breadcrumbs. Cloud storage is now the only source.`);
}

async function filesStatus(args: string[]) {
  const dir = args[0] || '.';

  let mirrored = 0, redirected = 0, local = 0;

  function scan(d: string) {
    for (const entry of readdirSync(d)) {
      if (entry.startsWith('.') && entry !== '.supabase') continue;
      const full = join(d, entry);
      if (entry === '.supabase') { mirrored++; continue; }
      let stat;
      try {
        stat = lstatSync(full);
      } catch {
        continue; // Broken symlink or permission error
      }
      if (stat.isSymbolicLink()) continue;
      if (stat.isDirectory()) scan(full);
      else if (entry.endsWith('.redirect.yaml') || entry.endsWith('.redirect')) redirected++;
      else if (!entry.endsWith('.md')) local++;
    }
  }
  scan(dir);

  console.log('File migration status:');
  console.log(`  Mirrored directories: ${mirrored}`);
  console.log(`  Redirected files: ${redirected}`);
  console.log(`  Local binary files: ${local}`);

  if (mirrored === 0 && redirected === 0 && local > 0) {
    console.log(`\n${local} local files. Run: gbrain files mirror <dir> to start migration.`);
  } else if (redirected > 0) {
    console.log(`\n${redirected} files redirected to storage. Run: gbrain files clean <dir> --yes to remove breadcrumbs.`);
  }
}

export function collectFiles(dir: string): string[] {
  const files: string[] = [];

  function walk(d: string) {
    for (const entry of readdirSync(d)) {
      if (entry.startsWith('.')) continue;
      if (entry === 'node_modules') continue;

      const full = join(d, entry);
      let stat;
      try {
        stat = lstatSync(full);
      } catch {
        continue; // Broken symlink or permission error
      }
      if (stat.isSymbolicLink()) continue;

      if (stat.isDirectory()) {
        walk(full);
      } else if (!entry.endsWith('.md')) {
        // Non-markdown files are candidates for storage
        files.push(full);
      }
    }
  }

  walk(dir);
  return files.sort();
}