Skip to content

armgit / papra

Public Imported from GitHub
0

The minimalistic document archiving platform.

7.7 KB
Raw Blame
// oxlint-disable no-console
import type { Database } from '../modules/app/database/database.types';
import process from 'node:process';
import { parseArgs, parseEnv } from 'node:util';
import * as p from '@clack/prompts';
import { castError } from '@corentinth/chisels';
import { createNoopLogger } from '@crowlog/logger';
import { count, eq } from 'drizzle-orm';
import { createIterator } from '../modules/app/database/database.usecases';
import { parseConfig } from '../modules/config/config';
import { documentsTable } from '../modules/documents/documents.table';
import { buildCreateDocumentStorageKey } from '../modules/documents/document-storage.usecases';
import { buildResolveStoragePatternContext } from '../modules/documents/storage-patterns/storage-pattern.usecases';
import { createOrganizationsRepository } from '../modules/organizations/organizations.repository';
import { createStorageService } from '../modules/storage/storage.services';
import { ensureBooleanArg } from './commons/args.utils';
import { runScriptWithDb } from './commons/run-script';

const envArgsToString = (args: (string | boolean)[] = []) =>
  args.filter((arg) => typeof arg === 'string').join('\n');

export async function migrateDocumentStorage({
  db,
  fromEnv,
  toEnv,
  deleteSource,
  isDryRun,
  prompts,
}: {
  db: Database;
  fromEnv: Record<string, string>;
  toEnv: Record<string, string>;
  deleteSource: boolean;
  isDryRun: boolean;
  prompts?: typeof p;
}) {
  const { config: fromConfig } = await parseConfig({ env: { ...process.env, ...fromEnv } });
  const { config: toConfig } = await parseConfig({ env: { ...process.env, ...toEnv } });

  const fromStorageService = createStorageService({
    storageConfig: fromConfig.documentsStorage,
    encryptionOptions: {
      isEncryptionEnabled: fromConfig.documentsStorage.encryption.isEncryptionEnabled,
      keyEncryptionKeys: fromConfig.documentsStorage.encryption.documentKeyEncryptionKeys,
    },
  });
  const toStorageService = createStorageService({
    storageConfig: toConfig.documentsStorage,
    encryptionOptions: {
      isEncryptionEnabled: toConfig.documentsStorage.encryption.isEncryptionEnabled,
      keyEncryptionKeys: toConfig.documentsStorage.encryption.documentKeyEncryptionKeys,
    },
  });

  prompts?.intro('Document Storage Migration');

  const createDocumentStorageKey = buildCreateDocumentStorageKey({
    storagePatternConfig: toConfig.documentsStorage.pattern,
    documentsStorageService: toStorageService,
    resolveStoragePatternContext: buildResolveStoragePatternContext({
      organizationsRepository: createOrganizationsRepository({ db }),
    }),
    logger: createNoopLogger(),
  });

  if (isDryRun) {
    prompts?.log.info(
      'This is a dry run, no actual migration will be performed.\nJust logging the documents that would be migrated.',
    );
  }

  const migrationErrors: {
    document: { id: string; originalName: string; originalStorageKey: string };
    error: unknown;
  }[] = [];

  // oxlint-disable-next-line typescript/no-useless-default-assignment -- defensive fallback for empty result set
  const [{ count: documentCount = 0 } = {}] = await db
    .select({ count: count() })
    .from(documentsTable);

  if (documentCount === 0) {
    prompts?.outro('No documents found in the database, nothing to migrate');
    return;
  }

  prompts?.log.info(`Found ${documentCount} documents in the database`);

  const query = db
    .select({
      id: documentsTable.id,
      originalStorageKey: documentsTable.originalStorageKey,
      organizationId: documentsTable.organizationId,
      originalName: documentsTable.originalName,
      documentDate: documentsTable.documentDate,
      createdAt: documentsTable.createdAt,
      mimeType: documentsTable.mimeType,
      fileEncryptionKeyWrapped: documentsTable.fileEncryptionKeyWrapped,
      fileEncryptionKekVersion: documentsTable.fileEncryptionKekVersion,
      fileEncryptionAlgorithm: documentsTable.fileEncryptionAlgorithm,
    })
    .from(documentsTable)
    .$dynamic();

  const documentIterator = createIterator({ query });

  const progress = prompts?.progress({
    style: 'heavy',
    max: documentCount,
  });

  progress?.start();

  for await (const document of documentIterator) {
    const {
      id,
      originalStorageKey: storageKey,
      organizationId,
      originalName,
      mimeType,
      fileEncryptionKeyWrapped,
      fileEncryptionKekVersion,
      fileEncryptionAlgorithm,
    } = document;

    progress?.message(`Migrating ${originalName} - ${storageKey}`);

    if (isDryRun) {
      await new Promise((resolve) => setTimeout(resolve, 10));
      progress?.advance();
      continue;
    }

    try {
      const { fileStream } = await fromStorageService.getFileStream({
        storageKey,
        fileEncryptionKeyWrapped,
        fileEncryptionKekVersion,
        fileEncryptionAlgorithm,
      });

      const { storageKey: newStorageKey } = await createDocumentStorageKey({
        documentId: id,
        organizationId,
        documentName: originalName,
        documentDate: document.documentDate,
        documentCreatedAt: document.createdAt,
      });

      const encryptionFields = await toStorageService.saveFile({
        fileStream,
        fileName: originalName,
        mimeType,
        storageKey: newStorageKey,
      });

      await db
        .update(documentsTable)
        .set({
          originalStorageKey: newStorageKey,
          fileEncryptionKeyWrapped: encryptionFields.fileEncryptionKeyWrapped,
          fileEncryptionKekVersion: encryptionFields.fileEncryptionKekVersion,
          fileEncryptionAlgorithm: encryptionFields.fileEncryptionAlgorithm,
        })
        .where(eq(documentsTable.id, id));

      if (deleteSource) {
        await fromStorageService.deleteFile({ storageKey });
      }
    } catch (error) {
      migrationErrors.push({ document, error });
    }

    progress?.advance();
  }

  progress?.stop(
    `Migration completed: ${documentCount - migrationErrors.length} succeeded, ${migrationErrors.length} failed.`,
  );

  if (migrationErrors.length > 0) {
    migrationErrors.forEach(({ document, error }) => {
      const { id, originalName, originalStorageKey } = document;
      prompts?.log.error(
        `- Document ID: ${id}, Name: ${originalName}, Storage Key: ${originalStorageKey}\n  Error: ${castError(error).message}`,
      );
    });
  }
}

await runScriptWithDb({ scriptName: 'migrate-document-storage', silent: true }, async ({ db }) => {
  const { values } = parseArgs({
    args: process.argv.slice(2),
    options: {
      'from': { type: 'string', multiple: true, default: [] },
      'to': { type: 'string', multiple: true, default: [] },
      'delete-source': { type: 'boolean', default: false },
      'dry-run': { type: 'boolean', default: false },
      'help': { type: 'boolean', short: 'h', default: false },
    },
    strict: false,
  });

  if (ensureBooleanArg(values.help)) {
    console.log(`
Usage: migrate-document-storage [options]

Options:
  --from <env>        Source environment variables (can be specified multiple times)
  --to <env>          Destination environment variables (can be specified multiple times)
  --delete-source     Delete source file after successful migration (backup recommended)
  --dry-run           Perform a dry run without actual migration, just logs
  -h, --help          Show this help message
      `);
    process.exit(0);
  }

  const fromEnv = parseEnv(envArgsToString(values.from)) as Record<string, string>;
  const toEnv = parseEnv(envArgsToString(values.to)) as Record<string, string>;
  const deleteSource = ensureBooleanArg(values['delete-source']);
  const isDryRun = ensureBooleanArg(values['dry-run']);

  await migrateDocumentStorage({
    db,
    fromEnv,
    toEnv,
    deleteSource,
    isDryRun,
    prompts: p,
  });
});