Skip to content

A free and open source instant messaging and VoIP chat app built for friends, groups, and communities.

TypeScript fluxerfluxerapp
8.2 KB
Raw Blame
// SPDX-License-Identifier: AGPL-3.0-or-later

import {createHash} from 'node:crypto';
import {alertSupport} from '@app/api/billing/BillingSupportAlert';
import type {BillingDocumentRepository} from '@app/api/billing_documents/BillingDocumentRepository';
import type {BillingDocumentRow} from '@app/api/database/types/BillingDocumentTypes';
import {Logger} from '@app/api/Logger';
import {extractId} from '@app/api/stripe/StripeUtils';
import type {BillingDocumentFlow} from '@pkgs/email/src/billing_documents/BillingDocumentContent';
import type {IKVProvider} from '@pkgs/kv_client/src/IKVProvider';
import type Stripe from 'stripe';
import {z} from 'zod';

export const LEGACY_IMPORT_TASK = 'importLegacyBillingDocuments';
const LEGACY_IMPORT_PAGE_SIZE = 20;
const LEGACY_DOCUMENT_KIND = 'stripe_invoice';
const LEGACY_SERIES = 'STRIPE';
const PROGRESS_KEY = 'billing_documents:legacy_import:progress';
const PDF_MAX_BYTES = 10 * 1024 * 1024;
const PDF_TIMEOUT_MS = 30_000;

export const LegacyImportPayloadSchema = z.object({
	run: z.string(),
	before: z.number().int(),
	dry_run: z.boolean(),
	starting_after: z.string().nullable(),
});

export type LegacyImportPayload = z.infer<typeof LegacyImportPayloadSchema>;

export interface LegacyImportCounts {
	scanned: number;
	imported: number;
	already_imported: number;
	would_import: number;
	no_account: number;
	fluxer_document: number;
	failed: number;
}

export interface LegacyImportProgress extends LegacyImportCounts {
	run: string;
	before: string;
	dry_run: boolean;
	cursor: string | null;
	done: boolean;
	updated_at: string;
}

export interface LegacyImportDeps {
	stripe: Stripe;
	repository: Pick<BillingDocumentRepository, 'findSource' | 'claimSource' | 'findDocument' | 'createDocument'>;
	findUserIdByCustomer: (customerId: string) => Promise<bigint | null>;
	nextId: () => Promise<bigint>;
	fetchPdf?: (url: string) => Promise<Buffer>;
}

export function emptyLegacyImportCounts(): LegacyImportCounts {
	return {scanned: 0, imported: 0, already_imported: 0, would_import: 0, no_account: 0, fluxer_document: 0, failed: 0};
}

function sha256(value: string | Uint8Array): string {
	return createHash('sha256').update(value).digest('hex');
}

function numericId(value: string | null | undefined): bigint | null {
	return value && /^\d+$/.test(value) ? BigInt(value) : null;
}

function invoiceMetadata(invoice: Stripe.Invoice): Record<string, string> {
	return {...invoice.metadata, ...invoice.parent?.subscription_details?.metadata};
}

function legacyInvoiceFlow(invoice: Stripe.Invoice): BillingDocumentFlow {
	const metadata = invoiceMetadata(invoice);
	if (metadata.operator_pass === 'true') return 'operator_pass';
	if (metadata.is_donation === 'true') return 'donation';
	if (metadata.is_gift === 'true') return 'gift';
	return 'plutonium';
}

async function downloadStripePdf(url: string): Promise<Buffer> {
	const response = await fetch(url, {signal: AbortSignal.timeout(PDF_TIMEOUT_MS)});
	if (!response.ok) {
		throw new Error(`Stripe invoice PDF download failed with ${response.status}`);
	}
	const bytes = Buffer.from(await response.arrayBuffer());
	if (bytes.byteLength > PDF_MAX_BYTES || bytes.subarray(0, 5).toString('latin1') !== '%PDF-') {
		throw new Error('Stripe invoice download is not a PDF');
	}
	return bytes;
}

async function resolveUserId(invoice: Stripe.Invoice, deps: LegacyImportDeps): Promise<bigint | null> {
	const metadata = invoiceMetadata(invoice);
	const fromMetadata = numericId(metadata.user_id) ?? numericId(metadata.operator_pass_purchaser_id);
	if (fromMetadata != null) {
		return fromMetadata;
	}
	const customerId = extractId(invoice.customer);
	return customerId ? deps.findUserIdByCustomer(customerId) : null;
}

function legacyRow(
	invoice: Stripe.Invoice & {id: string},
	documentId: bigint,
	userId: bigint,
	pdf: Buffer,
): BillingDocumentRow {
	const issuedAt = new Date((invoice.status_transitions?.finalized_at ?? invoice.created) * 1000);
	const number = invoice.number ?? invoice.id;
	const references = {stripeInvoiceId: invoice.id, customerId: extractId(invoice.customer)};
	const content = JSON.stringify({
		kind: LEGACY_DOCUMENT_KIND,
		number,
		issuedAt: issuedAt.toISOString(),
		currency: invoice.currency.toUpperCase(),
		totalMinor: invoice.total,
		references,
	});
	return {
		document_id: documentId,
		series: LEGACY_SERIES,
		number: 0n,
		display_number: number,
		kind: LEGACY_DOCUMENT_KIND,
		flow: legacyInvoiceFlow(invoice),
		layout: 'stripe',
		issued_at: issuedAt,
		supply_date: null,
		period_start: null,
		period_end: null,
		currency: invoice.currency.toUpperCase(),
		total_minor: BigInt(invoice.total),
		tax_minor: BigInt((invoice.total_taxes ?? []).reduce((sum, tax) => sum + tax.amount, 0)),
		tax_lines: '[]',
		sek_tax_minor: null,
		fx_rate: null,
		fx_date: null,
		buyer: JSON.stringify({email: invoice.customer_email ?? null, name: invoice.customer_name ?? null}),
		seller: '{}',
		language: 'en',
		original_document_id: null,
		original_number: null,
		reason: null,
		source_key: `li:${invoice.id}`,
		stripe_refs: JSON.stringify(references),
		user_id: userId,
		donor_email_hash: null,
		content,
		content_sha256: sha256(content),
		pdf,
		pdf_sha256: sha256(pdf),
		render_version: 'stripe',
		emailed_at: null,
		archived_at: null,
	};
}

async function importInvoice(
	invoice: Stripe.Invoice,
	dryRun: boolean,
	deps: LegacyImportDeps,
	counts: LegacyImportCounts,
): Promise<void> {
	const invoiceId = invoice.id;
	if (!invoiceId || invoice.status !== 'paid' || invoice.total <= 0 || !invoice.invoice_pdf) {
		return;
	}
	counts.scanned++;
	if (await deps.repository.findSource(`in:${invoiceId}`)) {
		counts.fluxer_document++;
		return;
	}
	const existing = await deps.repository.findSource(`li:${invoiceId}`);
	if (existing && (await deps.repository.findDocument(existing.document_id))) {
		counts.already_imported++;
		return;
	}
	const userId = await resolveUserId(invoice, deps);
	if (userId == null) {
		counts.no_account++;
		return;
	}
	if (dryRun) {
		counts.would_import++;
		return;
	}
	const pdf = await (deps.fetchPdf ?? downloadStripePdf)(invoice.invoice_pdf);
	const claimed = await deps.repository.claimSource(`li:${invoiceId}`, await deps.nextId());
	await deps.repository.createDocument(legacyRow({...invoice, id: invoiceId}, claimed.document_id, userId, pdf), {
		indexMonth: false,
	});
	counts.imported++;
}

export async function importLegacyInvoicePage(
	payload: LegacyImportPayload,
	deps: LegacyImportDeps,
): Promise<{counts: LegacyImportCounts; cursor: string | null; hasMore: boolean}> {
	const page = await deps.stripe.invoices.list({
		status: 'paid',
		created: {lt: payload.before},
		limit: LEGACY_IMPORT_PAGE_SIZE,
		...(payload.starting_after ? {starting_after: payload.starting_after} : {}),
	});
	const counts = emptyLegacyImportCounts();
	for (const invoice of page.data) {
		try {
			await importInvoice(invoice, payload.dry_run, deps, counts);
		} catch (error) {
			counts.failed++;
			Logger.error(
				{error, invoiceId: invoice.id, billingDocumentAlert: true},
				'Could not import a Stripe invoice PDF into billing history; run the import again to retry it',
			);
			await alertSupport({
				dedupeKey: `legacy-import:${payload.run}`,
				subject: 'Legacy Stripe invoice import had failures',
				summary: `At least one Stripe invoice PDF could not be imported into billing history in import run ${payload.run}. The import progress shows how many failed. Run the import again to retry them.`,
				facts: {invoiceId: invoice.id, error},
			});
		}
	}
	return {counts, cursor: page.data.at(-1)?.id ?? payload.starting_after, hasMore: page.has_more};
}

export async function readLegacyImportProgress(kv: Pick<IKVProvider, 'get'>): Promise<LegacyImportProgress | null> {
	const raw = await kv.get(PROGRESS_KEY);
	return raw ? (JSON.parse(raw) as LegacyImportProgress) : null;
}

export async function writeLegacyImportProgress(
	kv: Pick<IKVProvider, 'set'>,
	progress: LegacyImportProgress,
): Promise<void> {
	await kv.set(PROGRESS_KEY, JSON.stringify(progress));
}

export function addLegacyImportCounts(
	progress: LegacyImportProgress,
	counts: LegacyImportCounts,
): LegacyImportProgress {
	const next = {...progress};
	for (const key of Object.keys(counts) as Array<keyof LegacyImportCounts>) {
		next[key] = progress[key] + counts[key];
	}
	return next;
}