Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

3.9 KB
Raw Blame
#!/usr/bin/env node
/**
 * Verify an export directory against its manifest before importing it.
 *
 * Catches the things that actually go wrong: a file that never finished
 * transferring (truncated gzip), a file that is silently missing, and a file
 * the manifest does not list (left over from an earlier export, and imported
 * anyway, since the importer reads the directory).
 *
 *   node verify-export.mjs ./op-export
 *
 * Exits non-zero if anything is wrong, so it chains: `node verify-export.mjs
 * ./op-export && node import-to-cloud.mjs --dir ./op-export ...`
 */
import {
  createReadStream,
  existsSync,
  readdirSync,
  readFileSync,
} from 'node:fs';
import path from 'node:path';
import { createInterface } from 'node:readline';
import { createGunzip } from 'node:zlib';

const dir = process.argv[2] ?? './op-export';
const manifestPath = path.join(dir, 'manifest.jsonl');

if (!existsSync(manifestPath)) {
  console.error(
    `❌ No manifest.jsonl in ${dir} — is that the export directory?`
  );
  process.exit(1);
}

async function countRows(file) {
  const stream = file.endsWith('.gz')
    ? createReadStream(file).pipe(createGunzip())
    : createReadStream(file);
  const lines = createInterface({
    input: stream,
    crlfDelay: Number.POSITIVE_INFINITY,
  });
  let rows = 0;
  for await (const line of lines) {
    if (line.trim()) {
      rows++;
    }
  }
  return rows;
}

const entries = readFileSync(manifestPath, 'utf8')
  .split('\n')
  .filter(Boolean)
  .map((line) => JSON.parse(line));

const problems = [];
const notes = [];
const totals = {};
let checked = 0;

for (const entry of entries) {
  const file = path.join(dir, entry.table, entry.file);

  if (!existsSync(file)) {
    problems.push(`MISSING   ${entry.table}/${entry.file}`);
    continue;
  }

  let rows;
  try {
    rows = await countRows(file);
  } catch (error) {
    // An interrupted transfer shows up here as an unexpected end of stream.
    problems.push(`CORRUPT   ${entry.table}/${entry.file} — ${error.message}`);
    continue;
  }

  // Fewer rows than expected means the file was cut short. More rows means
  // traffic arrived between the export's counting pass and its reading pass —
  // the file is complete, the manifest's estimate was just stale. Only the
  // first case is a problem.
  if (rows < entry.rows) {
    problems.push(
      `TRUNCATED ${entry.table}/${entry.file} — manifest says ${entry.rows}, file has ${rows}`
    );
    continue;
  }

  if (rows > entry.rows) {
    notes.push(
      `${entry.table}/${entry.file} — ${rows - entry.rows} row(s) beyond the manifest (written while the export ran)`
    );
  }

  totals[entry.table] = (totals[entry.table] ?? 0) + rows;
  checked++;
}

const listed = new Set(entries.map((entry) => `${entry.table}/${entry.file}`));
for (const table of readdirSync(dir, { withFileTypes: true })) {
  if (!table.isDirectory()) {
    continue;
  }
  for (const file of readdirSync(path.join(dir, table.name))) {
    if (!listed.has(`${table.name}/${file}`)) {
      problems.push(`UNLISTED  ${table.name}/${file} — not in manifest.jsonl`);
    }
  }
}

console.log(`Checked ${checked}/${entries.length} files in ${dir}\n`);
for (const [table, rows] of Object.entries(totals)) {
  console.log(`   ${table.padEnd(24)} ${String(rows).padStart(10)} rows`);
}

if (notes.length > 0) {
  console.log(`\nℹ️  ${notes.length} file(s) grew during the export:`);
  for (const note of notes) {
    console.log(`   ${note}`);
  }
}

if (problems.length > 0) {
  console.error(`\n❌ ${problems.length} problem(s):`);
  for (const problem of problems) {
    console.error(`   ${problem}`);
  }
  console.error(
    '\n   Re-run your rsync — it will fetch just the bad files. Delete UNLISTED'
  );
  console.error('   files, or re-export into an empty directory.');
  process.exit(1);
}

console.log(
  '\n✅ Export is complete — no missing, truncated, corrupt or unlisted files.'
);