Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

7.6 KB
Raw Blame
import {
  CopyObjectCommand,
  HeadObjectCommand,
  ListObjectsV2Command,
  S3Client,
} from '@aws-sdk/client-s3';
import {
  appendFileSync,
  createReadStream,
  existsSync,
  readFileSync,
  writeFileSync,
} from 'fs';
import { createInterface } from 'readline';

const S3_BUCKET = process.env.S3_BUCKET;
const DRY_RUN = process.env.DRY_RUN === 'true';
const PREFIX = process.env.PREFIX ?? 'macro|';
const CONCURRENCY = parseInt(process.env.CONCURRENCY ?? '20', 10);
const PAGE_SIZE = parseInt(process.env.PAGE_SIZE ?? '100', 10);
const LIMIT = process.env.LIMIT ? parseInt(process.env.LIMIT, 10) : undefined;
const USER = process.env.USER_PREFIX;
const DOCUMENT_ID = process.env.DOCUMENT_ID;
const KEYS_FILE = process.env.KEYS_FILE;
const RESET = process.env.RESET === 'true';

if (!S3_BUCKET) {
  console.error('S3_BUCKET is required');
  process.exit(1);
}

const s3 = new S3Client({});

const CURSOR_FILE = `cursor-${S3_BUCKET}.txt`;
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const LOG_FILE = `migration-${S3_BUCKET}-${timestamp}.log`;
const COPIED_KEYS_FILE = `copied-keys-${S3_BUCKET}-${timestamp}.txt`;

function log(message: string) {
  const line = `[${new Date().toISOString()}] ${message}`;
  console.log(line);
  appendFileSync(LOG_FILE, line + '\n');
}

function loadCursor(): string | undefined {
  if (RESET) {
    clearCursor();
    return undefined;
  }
  if (USER || DOCUMENT_ID) return undefined;
  if (!existsSync(CURSOR_FILE)) return undefined;
  const cursor = readFileSync(CURSOR_FILE, 'utf-8').trim();
  if (!cursor) return undefined;
  log(`Resuming from cursor: ${cursor}`);
  return cursor;
}

function saveCursor(lastKey: string) {
  if (DRY_RUN || USER || DOCUMENT_ID) return;
  writeFileSync(CURSOR_FILE, lastKey + '\n');
}

function clearCursor() {
  if (existsSync(CURSOR_FILE)) {
    writeFileSync(CURSOR_FILE, '');
  }
}

// Matches keys in the format: {owner}/{uuid_v4}/{version_id}.{extension}
// e.g. macro|[email protected]/12f9a0ac-d445-45e3-94c1-5e8c02f0a6d8/564457.pdf
const UUID_V4 = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}';
const VERSION_WITH_EXT_REGEX = new RegExp(`${UUID_V4}/(\\d+)\\..+$`);
const SKIP_PATTERNS = [
  /converted\.pdf$/,
  /^temp_files\//,
  /^ONBOARDING_DOCUMENTS\//,
];

interface Stats {
  scanned: number;
  copied: number;
  skipped: number;
  missing: number;
  errors: number;
}

const stats: Stats = {
  scanned: 0,
  copied: 0,
  skipped: 0,
  missing: 0,
  errors: 0,
};

function buildPrefix(): string | undefined {
  if (USER && DOCUMENT_ID) return `${USER}/${DOCUMENT_ID}/`;
  if (USER) return `${USER}/`;
  return PREFIX || undefined;
}

function stripExtension(key: string): string {
  // Strip everything after the version_id (handles multipart extensions like .js.map)
  return key.replace(/(\d+)\..+$/, '$1');
}

function shouldProcess(key: string): boolean {
  if (!VERSION_WITH_EXT_REGEX.test(key)) return false;
  return !SKIP_PATTERNS.some((p) => p.test(key));
}

async function exists(key: string): Promise<boolean> {
  try {
    await s3.send(new HeadObjectCommand({ Bucket: S3_BUCKET, Key: key }));
    return true;
  } catch (err: any) {
    if (err?.$metadata?.httpStatusCode === 404 || err?.name === 'NotFound') {
      return false;
    }
    throw err;
  }
}

async function copyKey(oldKey: string, newKey: string): Promise<void> {
  if (await exists(newKey)) {
    if (DRY_RUN && (USER || DOCUMENT_ID))
      log(`[dry run] SKIP (exists): ${newKey}`);
    stats.skipped++;
    return;
  }

  if (DRY_RUN) {
    if (USER || DOCUMENT_ID) log(`[dry run] ${oldKey} -> ${newKey}`);
    stats.copied++;
    return;
  }

  try {
    await s3.send(
      new CopyObjectCommand({
        Bucket: S3_BUCKET,
        CopySource: `${S3_BUCKET}/${oldKey.split('/').map(encodeURIComponent).join('/')}`,
        Key: newKey,
      })
    );
    appendFileSync(COPIED_KEYS_FILE, oldKey + '\n');
    stats.copied++;
  } catch (err) {
    log(`ERROR copying ${oldKey}: ${err}`);
    stats.errors++;
  }
}

async function processBatch(keys: string[]): Promise<void> {
  const tasks = keys.map((key) => copyKey(key, stripExtension(key)));
  await Promise.all(tasks);
}

async function migrateFromFile(filePath: string) {
  log(`Reading keys from: ${filePath}`);

  const rl = createInterface({
    input: createReadStream(filePath),
    crlfDelay: Infinity,
  });

  let batch: string[] = [];
  for await (const line of rl) {
    const key = line.trim();
    if (!key) continue;
    stats.scanned++;
    batch.push(key);

    if (batch.length >= CONCURRENCY) {
      if (LIMIT !== undefined && stats.copied + stats.skipped >= LIMIT) break;
      await processBatch(batch);
      batch = [];

      if (stats.scanned % 1000 === 0) {
        log(
          `... scanned=${stats.scanned} copied=${stats.copied} skipped=${stats.skipped}`
        );
      }
    }
  }

  if (batch.length > 0) await processBatch(batch);
}

async function migrateFromScan() {
  const startAfter = loadCursor();

  let continuationToken: string | undefined;

  do {
    let response;
    try {
      response = await s3.send(
        new ListObjectsV2Command({
          Bucket: S3_BUCKET,
          Prefix: buildPrefix(),
          MaxKeys: PAGE_SIZE,
          EncodingType: 'url',
          ContinuationToken: continuationToken,
          ...(startAfter && !continuationToken
            ? { StartAfter: startAfter }
            : {}),
        })
      );
    } catch (err) {
      log(`ERROR listing objects: ${err}`);
      throw err;
    }

    const keys = (response.Contents ?? [])
      .filter((obj) => obj.Key)
      .map((obj) => decodeURIComponent(obj.Key!));

    stats.scanned += keys.length;

    let toMigrate = keys.filter(shouldProcess);

    if (LIMIT !== undefined) {
      const remaining = LIMIT - stats.copied - stats.skipped - stats.missing;
      if (remaining <= 0) break;
      toMigrate = toMigrate.slice(0, remaining);
    }

    // Process in concurrent batches
    for (let i = 0; i < toMigrate.length; i += CONCURRENCY) {
      const batch = toMigrate.slice(i, i + CONCURRENCY);
      await processBatch(batch);
    }

    // Save cursor to last scanned key for resumability
    const lastKey = keys[keys.length - 1];
    if (lastKey) saveCursor(lastKey);

    continuationToken = response.IsTruncated
      ? response.NextContinuationToken
      : undefined;

    if (continuationToken) {
      log(
        `... scanned=${stats.scanned} copied=${stats.copied} skipped=${stats.skipped}`
      );
    }
  } while (continuationToken);

  // Clear cursor on successful completion
  if (!DRY_RUN && !USER && !DOCUMENT_ID) clearCursor();
}

async function main() {
  log('=== Migrating S3 keys to extensionless convention ===');
  log(`Bucket: ${S3_BUCKET}`);
  if (KEYS_FILE) log(`Keys file: ${KEYS_FILE}`);
  if (!KEYS_FILE) log(`Prefix: ${PREFIX || '<all>'}`);
  log(`Concurrency: ${CONCURRENCY}`);
  if (USER) log(`User: ${USER}`);
  if (DOCUMENT_ID) log(`Document: ${DOCUMENT_ID}`);
  if (DRY_RUN) log(`=== DRY RUN MODE ===${LIMIT ? ` (limit: ${LIMIT})` : ''}`);
  log(`Log file: ${LOG_FILE}`);
  if (!DRY_RUN) log(`Copied keys file: ${COPIED_KEYS_FILE}`);

  if (KEYS_FILE) {
    await migrateFromFile(KEYS_FILE);
  } else {
    await migrateFromScan();
  }

  log('=== Migration complete ===');
  log(`Scanned: ${stats.scanned}`);
  log(`Copied:  ${stats.copied}`);
  log(`Skipped: ${stats.skipped} (extensionless key already exists)`);
  log(`Missing: ${stats.missing} (source key not found)`);
  log(`Errors:  ${stats.errors}`);
}

process.on('unhandledRejection', (err) => {
  log(`Unhandled rejection: ${err}`);
  process.exit(1);
});

main().catch((err) => {
  log(`Fatal error: ${err}`);
  process.exit(1);
});