Skip to content

LIBERATED AI CHAT

TypeScript
21.6 KB
Raw Blame
#!/usr/bin/env node
/**
 * Compact legacy telemetry/ files into monthly archives under archive/legacy/.
 *
 * Usage:
 *   node scripts/compact-hf-telemetry.mjs                  # dry-run (default)
 *   node scripts/compact-hf-telemetry.mjs --apply          # commit archives + delete legacy
 *
 * Requires: HF_TOKEN env var with write access to the dataset.
 *
 * Safety guarantees:
 *   - Dry-run is default; --apply required for any writes
 *   - FAIL-CLOSED: aborts on ANY download failure
 *   - Paginates the full file listing (HF API returns max 1000/page)
 *   - RESUMABLE: derives a stable run ID from source file list hash;
 *     stores archives under archive/legacy/<runId>/. On rerun after partial
 *     deletion, detects existing manifest and resumes deletion from the
 *     manifest's original source list — never rebuilds archives from a
 *     reduced file set.
 *   - Verifies ALL uploaded artifacts (archives + manifest + invalid-records)
 *     by re-downloading and checking checksums
 *   - Batches delete operations (max 50 per HF commit)
 *   - Preserves FULL invalid lines in a separate artifact
 *   - Post-deletion relist confirms all manifest sources are gone
 */

const HF_API = 'https://huggingface.co/api';
const DATASET_REPO = process.env.HF_DATASET_REPO || 'pliny-the-prompter/g0dm0d3';
const BRANCH = process.env.HF_DATASET_BRANCH || 'main';
const TOKEN = process.env.HF_TOKEN;
const APPLY = process.argv.includes('--apply');
const DELETE_BATCH_SIZE = 50;

if (!TOKEN) {
  console.error('ERROR: HF_TOKEN environment variable required');
  process.exit(1);
}

const headers = { Authorization: `Bearer ${TOKEN}` };

// ── Paginated file listing ──────────────────────────────────────────
async function listAllFiles(prefix, recursive = false) {
  const allFiles = [];
  let cursor = null;

  while (true) {
    const params = new URLSearchParams();
    if (cursor) params.set('cursor', cursor);
    if (recursive) params.set('recursive', 'true');
    const url = `${HF_API}/datasets/${DATASET_REPO}/tree/${BRANCH}/${prefix}?${params}`;
    const res = await fetch(url, { headers });
    if (!res.ok) throw new Error(`List failed (${res.status}): ${await res.text()}`);

    const page = await res.json();
    if (!Array.isArray(page) || page.length === 0) break;

    allFiles.push(...page);
    console.log(`  Listed ${allFiles.length} entries so far...`);

    const linkHeader = res.headers.get('link') || '';
    const nextMatch = linkHeader.match(/<[^>]*[?&]cursor=([^&>]+)[^>]*>;\s*rel="next"/);
    if (nextMatch) {
      cursor = decodeURIComponent(nextMatch[1]);
    } else {
      break;
    }
  }

  return allFiles;
}

async function downloadFile(path) {
  const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`;
  const res = await fetch(url, { headers });
  if (!res.ok) throw new Error(`Download failed (${res.status}): ${path}`);
  return res.text();
}

async function fileExists(path) {
  const url = `https://huggingface.co/datasets/${DATASET_REPO}/resolve/${BRANCH}/${path}`;
  const res = await fetch(url, { method: 'HEAD', headers });
  return res.ok;
}

async function sha256(text) {
  const data = new TextEncoder().encode(text);
  const buf = await crypto.subtle.digest('SHA-256', data);
  return [...new Uint8Array(buf)].map(b => b.toString(16).padStart(2, '0')).join('');
}

function utf8ToBase64(str) {
  const bytes = new TextEncoder().encode(str);
  let binStr = '';
  for (let i = 0; i < bytes.length; i++) binStr += String.fromCharCode(bytes[i]);
  return btoa(binStr);
}

async function commitFiles(operations, summary) {
  const url = `${HF_API}/datasets/${DATASET_REPO}/commit/${BRANCH}`;
  const lines = [
    JSON.stringify({ key: 'header', value: { summary } }),
    ...operations.map(op => JSON.stringify({ key: op.key, value: op.value })),
  ];
  const res = await fetch(url, {
    method: 'POST',
    headers: { ...headers, 'Content-Type': 'application/x-ndjson' },
    body: lines.join('\n'),
  });
  if (!res.ok) throw new Error(`Commit failed (${res.status}): ${(await res.text()).slice(0, 500)}`);
  return res.json();
}

// ── Derive stable run ID from the sorted source file list ───────────
async function deriveRunId(sourceFiles) {
  const pathList = sourceFiles.map(f => f.path).sort().join('\n');
  const hash = await sha256(pathList);
  return hash.slice(0, 12);
}

async function main() {
  console.log(`Mode: ${APPLY ? 'APPLY (will write)' : 'DRY-RUN (read-only)'}`);
  console.log(`Repo: ${DATASET_REPO} (branch: ${BRANCH})`);
  console.log('');

  // ── Step 1: Paginated listing ─────────────────────────────────────
  console.log('Step 1: Listing ALL legacy telemetry/ files (paginated)...');
  const allEntries = await listAllFiles('telemetry');
  const jsonlFiles = allEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
  console.log(`  Total entries: ${allEntries.length}`);
  console.log(`  JSONL files: ${jsonlFiles.length}`);

  if (jsonlFiles.length === 0) {
    console.log('  Nothing to compact. Done.');
    return;
  }

  // ── Step 2: Check for existing manifest (resume detection) ────────
  // Derive run ID from the FULL original source list. If a prior run
  // already archived everything, its manifest contains the original
  // source paths — we can resume deletion without re-downloading.
  const currentRunId = await deriveRunId(jsonlFiles);
  const runDir = `archive/legacy/${currentRunId}`;
  const manifestPath = `${runDir}/manifest.json`;

  console.log('');
  console.log(`Step 2: Checking for existing manifest (run ID: ${currentRunId})...`);

  // Also check if a DIFFERENT run's manifest exists (partial deletion scenario)
  let archiveLegacyEntries = [];
  try {
    archiveLegacyEntries = await listAllFiles('archive/legacy', true);
  } catch (err) {
    if (!/404|EntryNotFound|does not exist/i.test(String(err.message))) throw err;
  }
  const existingManifests = archiveLegacyEntries.filter(
    f => f.type === 'file' && f.path.endsWith('/manifest.json')
  );

  let resumeManifest = null;

  if (existingManifests.length > 0) {
    // Check each manifest to see if it contains source files that are now missing
    // (meaning deletion already started from that run)
    for (const mf of existingManifests) {
      try {
        const mfContent = await downloadFile(mf.path);
        const mfData = JSON.parse(mfContent);
        if (mfData.sourceFilePaths && Array.isArray(mfData.sourceFilePaths)) {
          const currentPaths = new Set(jsonlFiles.map(f => f.path));
          const manifestPaths = new Set(mfData.sourceFilePaths);

          // If manifest has paths that no longer exist, deletion already started
          const alreadyDeleted = mfData.sourceFilePaths.filter(p => !currentPaths.has(p));
          if (alreadyDeleted.length > 0) {
            console.log(`  Found prior manifest at ${mf.path}`);
            console.log(`    Original sources: ${mfData.sourceFilePaths.length}`);
            console.log(`    Already deleted: ${alreadyDeleted.length}`);
            console.log(`    Remaining: ${mfData.sourceFilePaths.length - alreadyDeleted.length}`);
            console.log('  RESUMING from prior manifest (archives already verified).');
            resumeManifest = mfData;
            break;
          }

          // If current files exactly match or are a superset, this is a fresh run
          // for the same data — use the existing archives if checksums match
          if (mf.path === manifestPath) {
            console.log(`  Found matching manifest at ${manifestPath}`);
            console.log('  Verifying existing archives...');
            let allValid = true;
            for (const arch of mfData.archives) {
              try {
                const content = await downloadFile(arch.path);
                const checksum = await sha256(content);
                if (checksum !== arch.checksum) {
                  console.error(`    CHECKSUM MISMATCH: ${arch.path}`);
                  allValid = false;
                  break;
                }
              } catch {
                console.error(`    MISSING: ${arch.path}`);
                allValid = false;
                break;
              }
            }
            if (allValid) {
              console.log('  All archives verified. Resuming deletion phase.');
              resumeManifest = mfData;
              break;
            } else {
              console.log('  Archives invalid — rebuilding from scratch.');
            }
          }
        }
      } catch (err) {
        console.warn(`  Could not parse manifest ${mf.path}: ${err.message}`);
      }
    }
  }

  // ── RESUME PATH: skip download/archive, go straight to deletion ───
  if (resumeManifest) {
    if (!APPLY) {
      const remaining = resumeManifest.sourceFilePaths.filter(
        p => jsonlFiles.some(f => f.path === p)
      );
      console.log('');
      console.log('DRY-RUN: Would resume deletion of remaining files.');
      console.log(`  Files still to delete: ${remaining.length}`);
      console.log(`  Batches: ${Math.ceil(remaining.length / DELETE_BATCH_SIZE)}`);
      return;
    }

    // Delete only files that still exist AND are in the manifest
    const manifestPathSet = new Set(resumeManifest.sourceFilePaths);
    const toDelete = jsonlFiles.filter(f => manifestPathSet.has(f.path));

    console.log('');
    console.log(`Resuming deletion: ${toDelete.length} files remaining...`);

    const batches = [];
    for (let i = 0; i < toDelete.length; i += DELETE_BATCH_SIZE) {
      batches.push(toDelete.slice(i, i + DELETE_BATCH_SIZE));
    }

    for (let i = 0; i < batches.length; i++) {
      const batch = batches[i];
      const deleteOps = batch.map(f => ({
        key: 'deletedFile',
        value: { path: f.path },
      }));
      await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`);
      console.log(`  Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`);
    }

    // Post-deletion verification
    console.log('');
    console.log('Post-deletion verification: relisting telemetry/...');
    const remaining = await listAllFiles('telemetry');
    const remainingJsonl = remaining.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
    const leftover = remainingJsonl.filter(f => manifestPathSet.has(f.path));
    if (leftover.length > 0) {
      console.error(`  WARNING: ${leftover.length} manifest sources still exist after deletion!`);
      console.error('  Paths:', leftover.map(f => f.path).slice(0, 10).join(', '));
      process.exit(1);
    }
    console.log(`  Confirmed: 0 manifest sources remain. ${remainingJsonl.length} non-manifest files present.`);

    console.log('');
    console.log('DONE (resumed). Legacy telemetry compacted successfully.');
    return;
  }

  // ── FRESH PATH: download, archive, verify, delete ─────────────────

  // ── Step 3: Download and parse ALL files (fail-closed) ────────────
  console.log('');
  console.log('Step 3: Downloading and parsing events (fail-closed)...');
  const monthBuckets = {};
  let totalEvents = 0;
  let totalSourceBytes = 0;
  let downloadFailures = 0;
  let invalidLines = 0;
  const invalidRecords = [];
  const failedFiles = [];

  for (let i = 0; i < jsonlFiles.length; i++) {
    const file = jsonlFiles[i];
    if (i % 200 === 0 || i === jsonlFiles.length - 1) {
      console.log(`  Progress: ${i + 1}/${jsonlFiles.length} files...`);
    }

    let content;
    try {
      content = await downloadFile(file.path);
    } catch (err) {
      downloadFailures++;
      failedFiles.push({ path: file.path, error: err.message });
      continue;
    }

    totalSourceBytes += new TextEncoder().encode(content).byteLength;
    const lines = content.split('\n').filter(l => l.trim());

    for (const line of lines) {
      try {
        const event = JSON.parse(line);
        const ts = event.timestamp;
        let monthKey;
        if (ts && typeof ts === 'number') {
          const d = new Date(ts);
          monthKey = `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, '0')}`;
        } else {
          monthKey = 'unknown';
        }
        if (!monthBuckets[monthKey]) monthBuckets[monthKey] = [];
        monthBuckets[monthKey].push(line);
        totalEvents++;
      } catch {
        invalidLines++;
        invalidRecords.push({ file: file.path, line });
      }
    }
  }

  console.log(`  Total events parsed: ${totalEvents}`);
  console.log(`  Total source bytes: ${(totalSourceBytes / 1024 / 1024).toFixed(2)} MiB`);
  console.log(`  Download failures: ${downloadFailures}`);
  console.log(`  Invalid lines: ${invalidLines}`);
  console.log(`  Month buckets: ${Object.keys(monthBuckets).sort().join(', ')}`);

  // ── FAIL-CLOSED: abort if any downloads failed ────────────────────
  if (downloadFailures > 0) {
    console.error('');
    console.error(`ABORT: ${downloadFailures} file(s) failed to download.`);
    console.error('Cannot guarantee data preservation. Fix network/auth issues and retry.');
    console.error('Failed files:', failedFiles.map(f => f.path).join(', '));
    process.exit(1);
  }

  // ── Step 4: Build archives ────────────────────────────────────────
  console.log('');
  console.log(`Step 4: Building monthly archives under ${runDir}/...`);
  const archives = [];
  for (const [month, events] of Object.entries(monthBuckets).sort()) {
    const content = events.join('\n') + '\n';
    const checksum = await sha256(content);
    const archivePath = `${runDir}/telemetry-${month}.jsonl`;
    const bytes = new TextEncoder().encode(content).byteLength;
    archives.push({ month, path: archivePath, events: events.length, bytes, checksum, content });
    console.log(`  ${archivePath}: ${events.length} events, ${(bytes / 1024).toFixed(0)} KiB, sha256:${checksum.slice(0, 16)}...`);
  }

  // ── Step 5: Verify counts ─────────────────────────────────────────
  const archiveEventTotal = archives.reduce((s, a) => s + a.events, 0);
  const sourceFileCount = jsonlFiles.length;

  console.log('');
  console.log('Step 5: Count verification');
  console.log(`  Source files examined: ${sourceFileCount}`);
  console.log(`  Events parsed from sources: ${totalEvents}`);
  console.log(`  Events in archives: ${archiveEventTotal}`);
  console.log(`  Invalid lines preserved: ${invalidLines}`);

  if (archiveEventTotal !== totalEvents) {
    console.error('  COUNT MISMATCH — aborting. No changes made.');
    process.exit(1);
  }
  console.log('  COUNTS MATCH.');

  // ── Build manifest (includes full source path list for resumability) ─
  const invalidRecordsContent = invalidRecords.length > 0
    ? JSON.stringify(invalidRecords, null, 2)
    : null;
  const invalidRecordsChecksum = invalidRecordsContent
    ? await sha256(invalidRecordsContent)
    : null;

  const manifest = {
    runId: currentRunId,
    timestamp: new Date().toISOString(),
    repo: DATASET_REPO,
    branch: BRANCH,
    sourceFiles: sourceFileCount,
    sourceBytes: totalSourceBytes,
    sourceFilePaths: jsonlFiles.map(f => f.path).sort(),
    totalEvents,
    invalidLines,
    invalidRecordCount: invalidRecords.length,
    invalidRecordsChecksum,
    archives: archives.map(a => ({
      path: a.path, month: a.month, events: a.events,
      bytes: a.bytes, checksum: a.checksum,
    })),
  };

  const manifestContent = JSON.stringify(manifest, null, 2);
  const manifestChecksum = await sha256(manifestContent);

  console.log('');
  console.log('Manifest:');
  console.log(manifestContent);

  if (!APPLY) {
    console.log('');
    console.log('DRY-RUN complete. Re-run with --apply to commit changes.');
    console.log(`Would upload ${archives.length} archive files + manifest + invalid-records.`);
    console.log(`Would delete ${sourceFileCount} legacy files in ${Math.ceil(sourceFileCount / DELETE_BATCH_SIZE)} batches.`);
    return;
  }

  // ── Step 6: Upload archives ───────────────────────────────────────
  console.log('');
  console.log('Step 6: Uploading archives...');

  // Check if archives already exist and match (idempotent upload)
  let archivesAlreadyExist = false;
  if (await fileExists(manifestPath)) {
    try {
      const existing = await downloadFile(manifestPath);
      const existingChecksum = await sha256(existing);
      if (existingChecksum === manifestChecksum) {
        console.log('  Archives already uploaded (manifest checksum matches). Skipping upload.');
        archivesAlreadyExist = true;
      }
    } catch { /* not found or parse error — upload fresh */ }
  }

  if (!archivesAlreadyExist) {
    const uploadOps = archives.map(a => ({
      key: 'file',
      value: { content: utf8ToBase64(a.content), path: a.path, encoding: 'base64' },
    }));
    uploadOps.push({
      key: 'file',
      value: {
        content: utf8ToBase64(manifestContent),
        path: manifestPath,
        encoding: 'base64',
      },
    });
    if (invalidRecordsContent) {
      uploadOps.push({
        key: 'file',
        value: {
          content: utf8ToBase64(invalidRecordsContent),
          path: `${runDir}/invalid-records.json`,
          encoding: 'base64',
        },
      });
    }

    await commitFiles(uploadOps, `[maintenance] Archive ${totalEvents} legacy telemetry events into ${archives.length} monthly files (run ${currentRunId})`);
    console.log('  Archives committed.');
  }

  // ── Step 7: Verify ALL uploaded artifacts ─────────────────────────
  console.log('');
  console.log('Step 7: Verifying uploaded artifacts...');

  // Verify manifest
  const dlManifest = await downloadFile(manifestPath);
  const dlManifestChecksum = await sha256(dlManifest);
  if (dlManifestChecksum !== manifestChecksum) {
    console.error(`  MANIFEST CHECKSUM MISMATCH!`);
    console.error(`    Expected: ${manifestChecksum}`);
    console.error(`    Got:      ${dlManifestChecksum}`);
    console.error('  ABORTING — will not delete sources.');
    process.exit(1);
  }
  console.log(`  ${manifestPath}: checksum verified`);

  // Verify each archive
  for (const archive of archives) {
    const downloaded = await downloadFile(archive.path);
    const remoteChecksum = await sha256(downloaded);
    if (remoteChecksum !== archive.checksum) {
      console.error(`  CHECKSUM MISMATCH for ${archive.path}!`);
      console.error(`    Expected: ${archive.checksum}`);
      console.error(`    Got:      ${remoteChecksum}`);
      console.error('  ABORTING — archives uploaded but legacy files NOT deleted.');
      process.exit(1);
    }
    console.log(`  ${archive.path}: checksum verified`);
  }

  // Verify invalid-records if present
  if (invalidRecordsContent) {
    const dlInvalid = await downloadFile(`${runDir}/invalid-records.json`);
    const dlInvalidChecksum = await sha256(dlInvalid);
    if (dlInvalidChecksum !== invalidRecordsChecksum) {
      console.error(`  INVALID-RECORDS CHECKSUM MISMATCH!`);
      console.error('  ABORTING — will not delete sources.');
      process.exit(1);
    }
    console.log(`  ${runDir}/invalid-records.json: checksum verified`);
  }

  console.log('  All artifacts verified.');

  // ── Step 8: Delete legacy files in batches ────────────────────────
  console.log('');
  console.log(`Step 8: Deleting ${sourceFileCount} legacy files in batches of ${DELETE_BATCH_SIZE}...`);
  const batches = [];
  for (let i = 0; i < jsonlFiles.length; i += DELETE_BATCH_SIZE) {
    batches.push(jsonlFiles.slice(i, i + DELETE_BATCH_SIZE));
  }

  for (let i = 0; i < batches.length; i++) {
    const batch = batches[i];
    const deleteOps = batch.map(f => ({
      key: 'deletedFile',
      value: { path: f.path },
    }));
    await commitFiles(deleteOps, `[maintenance] Remove legacy telemetry batch ${i + 1}/${batches.length} (${batch.length} files)`);
    console.log(`  Batch ${i + 1}/${batches.length}: deleted ${batch.length} files`);
  }

  // ── Step 9: Post-deletion verification ────────────────────────────
  console.log('');
  console.log('Step 9: Post-deletion verification...');
  const postDeleteEntries = await listAllFiles('telemetry');
  const postDeleteJsonl = postDeleteEntries.filter(f => f.type === 'file' && f.path.endsWith('.jsonl'));
  const sourcePathSet = new Set(jsonlFiles.map(f => f.path));
  const leftover = postDeleteJsonl.filter(f => sourcePathSet.has(f.path));

  if (leftover.length > 0) {
    console.error(`  WARNING: ${leftover.length} source files still exist after deletion!`);
    console.error('  First 10:', leftover.slice(0, 10).map(f => f.path).join(', '));
    console.error('  Rerun with --apply to retry deletion of remaining files.');
    process.exit(1);
  }
  console.log(`  Confirmed: all ${sourceFileCount} manifest sources deleted.`);
  if (postDeleteJsonl.length > 0) {
    console.log(`  Note: ${postDeleteJsonl.length} non-manifest JSONL files remain (new incoming data).`);
  }

  console.log('');
  console.log('DONE. Legacy telemetry compacted successfully.');
  console.log(`  Run ID: ${currentRunId}`);
  console.log(`  ${archives.length} monthly archives in ${runDir}/`);
  console.log(`  ${sourceFileCount} legacy files removed`);
  console.log(`  ${totalEvents} events preserved`);
}

main().catch(err => {
  console.error('Fatal error:', err);
  process.exit(1);
});