Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

11.4 KB
Raw Blame
// Usage:
// bun run scripts/generate-api-schema.ts <service-name> // generate for specific service
// bun run scripts/generate-api-schema.ts               // generate for all services
// bun run scripts/generate-api-schema.ts --check       // verify types are up to date (for CI)
// bun run scripts/generate-api-schema.ts --print-cargo-bins // print the `--bin` args phase 1 builds (for CI)
//
// This script generates OpenAPI schemas by running the local Rust binaries
// instead of fetching from deployed services.

import * as path from "node:path";
import { $, write } from "bun";
import { type Service, services } from "./services";

const elapsed = (start: number) =>
	`${((performance.now() - start) / 1000).toFixed(1)}s`;

// Map API names to their OpenAPI binary prefixes (several share the harness crate).
const serviceToCrate: Record<string, string> = {
	"cloud-storage": "document_storage_service",
	"properties-service": "properties_service",
	"document-cognition": "agent_harness_cognition",
	"auth-service": "authentication_service",
	"notification-service": "notification_service",
	"static-files": "static_file_service",
	"connection-gateway": "connection_gateway",
	"contacts-service": "contacts_service",
	"unfurl-service": "unfurl_service",
	"agent-harness": "agent_harness_service",
	"calendar-service": "calendar_service",
	"email-service": "email_service",
	"search-service": "search_service",
	"scheduled-action": "scheduled_action"
};

const getRustWorkspaceDir = () =>
	path.resolve(import.meta.dirname, "../../..");
const getServiceClientsDir = () =>
	path.resolve(import.meta.dirname, "../src/lib/service-clients");
// Parse arguments
const getTargetServices = () =>
	process.argv
		.slice(2)
		.filter((arg) => arg !== "--check" && arg !== "--print-cargo-bins");

const openApiBinArgs = (crateNames: string[]) =>
	crateNames.flatMap((crate) => ["--bin", `${crate}_openapi`]);

// Build all OpenAPI binaries in a single cargo invocation (cargo parallelizes internally).
async function buildOpenApiBinaries(
	crateNames: string[],
	{ rustWorkspaceDir = getRustWorkspaceDir() } = {},
): Promise<void> {
	if (crateNames.length === 0) return;

	const binArgs = openApiBinArgs(crateNames);

	console.log(`Building ${crateNames.length} OpenAPI binaries...`);
	console.log(`cargo build args: ${binArgs.join(" ")}`);
	await $`cd ${rustWorkspaceDir} && SQLX_OFFLINE=true cargo build ${binArgs}`;
	console.log("Build complete.\n");
}

// Run a named binary from the debug directory with a timeout and stderr capture.
// Throws with stderr content on non-zero exit or timeout so CI failures aren't silent.
const BINARY_TIMEOUT_MS = 120_000;

async function runBinary(
	binaryName: string,
	rustWorkspaceDir = getRustWorkspaceDir(),
): Promise<string> {
	const binaryPath = process.env.OPENAPI_BINS_DIR
		? path.join(process.env.OPENAPI_BINS_DIR, binaryName)
		: path.join(rustWorkspaceDir, "target", "debug", binaryName);
	const proc = Bun.spawn([binaryPath], { stdout: "pipe", stderr: "pipe" });
	const TIMEOUT_SENTINEL = Symbol("timeout");
	let timeout: ReturnType<typeof setTimeout> | undefined;
	const timeoutPromise = new Promise<typeof TIMEOUT_SENTINEL>((resolve) => {
		timeout = setTimeout(() => {
			proc.kill();
			resolve(TIMEOUT_SENTINEL);
		}, BINARY_TIMEOUT_MS);
	});
	const completionPromise = Promise.all([
		new Response(proc.stdout).text(),
		new Response(proc.stderr).text(),
		proc.exited,
	]);
	const result = await Promise.race([completionPromise, timeoutPromise]);
	clearTimeout(timeout);
	if (result === TIMEOUT_SENTINEL) {
		// After the kill, the in-flight stream reads should complete quickly — grab
		// whatever stderr was buffered, but don't wait forever if they don't.
		const drained = await Promise.race([
			completionPromise.catch(() => null),
			new Promise<null>((resolve) => setTimeout(() => resolve(null), 1_000)),
		]);
		const stderr = drained ? drained[1] : "";
		throw new Error(
			`${binaryName} timed out after ${BINARY_TIMEOUT_MS}ms\nstderr:\n${stderr}`,
		);
	}
	const [stdout, stderr, exitCode] = result;
	if (exitCode !== 0) {
		throw new Error(
			`${binaryName} exited with code ${exitCode}\nstderr:\n${stderr}`,
		);
	}
	return stdout;
}

const runOpenApiBinary = (
	crateName: string,
	rustWorkspaceDir = getRustWorkspaceDir(),
) => runBinary(`${crateName}_openapi`, rustWorkspaceDir);

const getServicesToProcess = (targetServices: string[]) => {
	// Figure out which services to process
	let servicesToProcess: Service[];
	if (targetServices.length > 0) {
		// Filter only those whose names match the arguments
		servicesToProcess = services.filter((service) =>
			targetServices.includes(service.name),
		);

		// If none matched, bail out
		if (servicesToProcess.length === 0) {
			console.error(
				`Error: No matching services found for [${targetServices.join(", ")}].
         Valid options are: ${services.map((s) => s.name).join(", ")}`,
			);
			process.exit(1);
		}
	} else {
		// If no arguments, process all
		servicesToProcess = services;
	}

	return servicesToProcess;
};

// Process a single service (assumes binary is already built)
const processService = async (
	service: Service,
	{ serviceClientsDir }: { serviceClientsDir: string },
) => {
	const crateName = serviceToCrate[service.name];
	if (!crateName) {
		console.error(`[${service.name}] No crate mapping found, skipping`);
		return { service: service.name, status: "skipped" as const };
	}

	try {
		const outputDir = path.resolve(import.meta.dirname, service.output);
		const generatedDir = path.resolve(outputDir, "generated");
		const openApiPath = path.join(outputDir, "openapi.json");

		const serviceStart = performance.now();

		console.log(`[${service.name}] Running OpenAPI binary...`);
		let stepStart = performance.now();
		const openApiJson = await runOpenApiBinary(crateName);
		console.log(
			`[${service.name}] OpenAPI binary finished (${elapsed(stepStart)})`,
		);

		// Remove existing generated dir
		await $`rm -rf ${generatedDir}`.quiet();

		// Write the OpenAPI JSON
		await write(openApiPath, openApiJson);
		console.log(`[${service.name}] Saved OpenAPI spec`);

		// Run orval to generate types
		stepStart = performance.now();
		await $`cd ${serviceClientsDir} && bun run orval --config orval.config.ts --project ${service.orvalKey}`;
		console.log(`[${service.name}] Orval finished (${elapsed(stepStart)})`);

		if (service.name === "notification-service") {
			await $`bun ${path.resolve(import.meta.dirname, "generate-push-schema.ts")}`;
		}

		// Orval bug workaround: with input.filters, excluded schemas are not
		// generated, but the schemas barrel still re-exports them. Drop barrel
		// exports whose target file does not exist.
		const schemasIndex = path.join(generatedDir, "schemas", "index.ts");
		if (await Bun.file(schemasIndex).exists()) {
			const lines = (await Bun.file(schemasIndex).text()).split("\n");
			const kept: string[] = [];
			for (const line of lines) {
				const target = line.match(/^export \* from ['"]\.\/(.+)['"];$/);
				const file = target
					? Bun.file(path.join(generatedDir, "schemas", `${target[1]}.ts`))
					: null;
				if (file && !(await file.exists())) continue;
				kept.push(line);
			}
			await write(schemasIndex, kept.join("\n"));
		}

		// Special handling for document-cognition: generate AI tool schema types.
		if (service.name === "document-cognition") {
			stepStart = performance.now();
			const appDir = path.resolve(import.meta.dirname, "..");
			await $`cd ${appDir} && bun scripts/generate-dcs-tools.ts`.quiet();
			console.log(
				`[${service.name}] DCS tool types generation finished (${elapsed(stepStart)})`,
			);
		}

		console.log(`[${service.name}] ✓ Done (total: ${elapsed(serviceStart)})`);
		return { service: service.name, status: "success" as const };
	} catch (error) {
		console.error(`[${service.name}] Failed:`, error);
		return { service: service.name, status: "failed" as const, error };
	}
};

async function main() {
	const serviceClientsDir = getServiceClientsDir();
	const checkMode = process.argv.includes("--check");
	const targetServices = getTargetServices();
	const servicesToProcess = getServicesToProcess(targetServices);

	// Get crate names for services that have mappings
	const crateNames = servicesToProcess
		.map((s) => serviceToCrate[s.name])
		.filter((crate): crate is string => !!crate);

	// CI builds these together with other schema binaries in one cargo
	// invocation, so the phase 1 build below finds them fresh.
	if (process.argv.includes("--print-cargo-bins")) {
		console.log(openApiBinArgs(crateNames).join(" "));
		return;
	}

	console.log(`\nProcessing ${servicesToProcess.length} service(s)...\n`);

	// Phase 1: Build all binaries (skipped when OPENAPI_BINS_DIR is set)
	const buildStart = performance.now();
	if (process.env.OPENAPI_BINS_DIR) {
		console.log(
			`Phase 1 (cargo build) skipped — using OPENAPI_BINS_DIR=${process.env.OPENAPI_BINS_DIR}`,
		);
	} else {
		await buildOpenApiBinaries(crateNames);
		console.log(`Phase 1 (cargo build) total: ${elapsed(buildStart)}`);
	}

	// Phase 2: Run binaries and generate TypeScript in parallel
	console.log("\nGenerating TypeScript clients...\n");
	const results = await Promise.all(
		servicesToProcess.map((service) =>
			processService(service, { serviceClientsDir }),
		),
	);

	// Summary report
	console.log("\nProcessing Summary:");
	const succeeded = results.filter((r) => r.status === "success");
	const failed = results.filter((r) => r.status === "failed");
	const skipped = results.filter((r) => r.status === "skipped");

	console.log(`Succeeded: ${succeeded.length}/${servicesToProcess.length}`);
	if (skipped.length > 0) {
		console.log(`Skipped: ${skipped.length}/${servicesToProcess.length}`);
	}
	if (failed.length > 0) {
		console.log(`Failed: ${failed.length}/${servicesToProcess.length}`);
		failed.forEach((result) => {
			console.error(`  - ${result.service}:`, result.error);
		});
		process.exit(1);
	}
	// On NixOS, the npm-installed biome binary doesn't work due to dynamic linking issues.
	// We detect NixOS and use the system biome instead.
	const isNixOS =
		process.env.NIX_PATH !== undefined ||
		((await Bun.file("/etc/os-release").exists()) &&
			(await Bun.file("/etc/os-release").text()).includes("NixOS"));
	const biomeStart = performance.now();
	console.log("\nRunning biome check...");
	if (isNixOS) {
		const systemBiomePath =
			await $`bash -c 'PATH=$(echo "$PATH" | tr ":" "\n" | grep -v node_modules | tr "\n" ":") which biome'`.text();
		await $`${systemBiomePath.trim()} check --write --unsafe src/lib/service-clients/`;
	} else {
		await $`biome check --write --unsafe src/lib/service-clients/`;
	}
	console.log(`Biome check finished (${elapsed(biomeStart)})`);

	// In check mode, verify no uncommitted changes
	if (checkMode) {
		const diff =
			await $`git diff --ignore-blank-lines ${serviceClientsDir}`.text();
		const untrackedFiles =
			await $`git ls-files --others --exclude-standard ${serviceClientsDir}`.text();

		if (diff.trim() || untrackedFiles.trim()) {
			console.error(
				"\n❌ Generated types are out of sync with Rust API definitions!",
			);
			console.error("The following files have changed:");
			if (diff.trim()) console.error(diff);
			if (untrackedFiles.trim()) console.error(`Untracked:\n${untrackedFiles}`);
			const gitStatus = await $`git status`.text();
			console.error("`git status` output:");
			console.error(gitStatus.trim());
			console.error("\nPlease run: bun run scripts/generate-api-schema.ts");
			console.error("Then commit the changes.");
			process.exit(1);
		}

		console.log("\n✓ Generated types are up to date");
	}
}

await main();