Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

6.1 KB
Raw Blame
/**
 * The scan orchestrator.
 *
 * One pass: enumerate the fleet, gather each repository's own claims, probe
 * those claims against the live internet, verify its packages against the npm
 * registry, and score the result. Progress is observable while it runs, because
 * a full fleet scan takes minutes and a spinner with no numbers is not a
 * loading state.
 */

import { config } from './config.js';
import { mapPool } from './pool.js';
import { listRepos, fetchReadme, fetchPackageJson, fetchPagesUrl, fetchRootEntries, rateLimit } from './github.js';
import { extractReadmeLinks, deploymentCandidates } from './extract-urls.js';
import { probeUrl } from './probe.js';
import { verifyPackages } from './registry.js';
import { scoreRepo, summarise } from './score.js';

/** Live progress for the dashboard's scanning state. */
export const progress = {
	running: false,
	startedAt: null,
	finishedAt: null,
	total: 0,
	done: 0,
	current: '',
	phase: 'idle',
	error: '',
	lastDurationMs: 0
};

const DOC_DIRS = new Set(['docs', 'doc', 'documentation', 'website', 'site']);

/**
 * Why a snapshot covers less than the whole fleet, in plain sentences.
 *
 * Shared by the CLI and the dashboard so the two can never disagree about the
 * cause. Both reasons can hold at once, in which case both are reported: an
 * operator who raises FLEET_MAX_REPOS without also raising the request budget
 * would otherwise fix half the problem and see no change.
 *
 * @param {object|null} snapshot
 * @returns {string[]} one sentence per reason, empty when the scan was complete
 */
export function partialReasons(snapshot) {
	if (!snapshot?.partial) return [];
	const out = [];
	const scanned = snapshot.summary?.repos ?? 0;
	const owned = snapshot.totalOwned ?? scanned;

	if (snapshot.partialReason === 'rate_limit') {
		const resets = snapshot.rateLimit?.resetAt ? `, which resets at ${snapshot.rateLimit.resetAt}` : '';
		const raise = snapshot.authenticated ? 'Some measurements are incomplete.' : 'Set GITHUB_TOKEN to raise the budget from 60 to 5000 requests an hour.';
		out.push(`The GitHub request budget ran out during this scan${resets}. ${raise}`);
	}
	if (scanned < owned) {
		out.push(`Only the ${scanned} most-starred of ${owned} repositories were scanned. Raise FLEET_MAX_REPOS to cover the whole fleet.`);
	}
	// A snapshot written before partialReason existed still reloads from disk and
	// still renders, so it gets an honest sentence rather than an empty banner.
	if (out.length === 0) out.push(`${scanned} of ${owned} repositories were scanned.`);
	return out;
}

async function gatherRepo(repo) {
	const [readme, manifest, entries, pagesUrl] = await Promise.all([
		fetchReadme(repo.fullName),
		fetchPackageJson(repo.fullName, repo.defaultBranch),
		fetchRootEntries(repo.fullName, repo.defaultBranch),
		repo.hasPages ? fetchPagesUrl(repo.fullName) : Promise.resolve('')
	]);

	const links = extractReadmeLinks(readme);
	const candidates = deploymentCandidates({
		repoName: repo.name,
		homepage: repo.homepage,
		pagesUrl,
		manifestHomepage: typeof manifest?.homepage === 'string' ? manifest.homepage : '',
		links
	});

	const candidateUrls = new Set(candidates.map((entry) => entry.url));
	// Check links that are not already covered by a deployment probe.
	const linkTargets = links
		.map((link) => link.url)
		.filter((url) => !candidateUrls.has(url))
		.slice(0, config.maxLinksPerRepo);

	const [deployments, linkResults, packages] = await Promise.all([
		mapPool(candidates, 4, async (candidate) => ({ ...(await probeUrl(candidate.url)), why: candidate.why })),
		mapPool(linkTargets, 4, (url) => probeUrl(url)),
		verifyPackages({ manifest, readme })
	]);

	return {
		...repo,
		pagesUrl,
		readmeBytes: Buffer.byteLength(readme, 'utf8'),
		hasDocsDir: entries.some((entry) => entry.type === 'dir' && DOC_DIRS.has(entry.name.toLowerCase())),
		packageName: manifest?.name || '',
		packageVersion: manifest?.version ? String(manifest.version) : '',
		deployments,
		links: linkResults,
		packages
	};
}

/**
 * Run one full fleet scan.
 * @returns {Promise<object>} the snapshot
 */
export async function runScan({ owner = config.owner, limit = config.maxRepos } = {}) {
	if (progress.running) throw new Error('a scan is already running');

	const startedAt = Date.now();
	Object.assign(progress, {
		running: true,
		startedAt: new Date(startedAt).toISOString(),
		finishedAt: null,
		total: 0,
		done: 0,
		current: '',
		phase: 'enumerating',
		error: ''
	});

	try {
		const all = await listRepos(owner);
		const selected = all.slice(0, limit);
		progress.total = selected.length;
		progress.phase = 'scanning';

		const scanned = await mapPool(selected, config.githubConcurrency, async (repo) => {
			progress.current = repo.name;
			try {
				const gathered = await gatherRepo(repo);
				const { score, grade, checks } = scoreRepo(gathered);
				return { ...gathered, score, grade, checks };
			} catch (error) {
				return { ...repo, score: null, grade: null, checks: [], scanError: String(error?.message || error).slice(0, 200) };
			} finally {
				progress.done += 1;
			}
		});

		// A capped scan and an exhausted rate-limit budget are both "partial" and
		// look identical in the data, but they are different problems with
		// different fixes. Record which one happened so the console never blames
		// GitHub for a limit this service set itself.
		const rateLimited = Boolean(rateLimit.exhausted);
		const capped = selected.length < all.length;

		const snapshot = {
			owner,
			generatedAt: new Date().toISOString(),
			durationMs: Date.now() - startedAt,
			partial: rateLimited || capped,
			partialReason: rateLimited ? 'rate_limit' : capped ? 'repo_cap' : '',
			totalOwned: all.length,
			authenticated: Boolean(config.githubToken),
			rateLimit: { limit: rateLimit.limit, remaining: rateLimit.remaining, resetAt: rateLimit.resetAt },
			summary: summarise(scanned),
			repos: scanned
		};

		progress.lastDurationMs = snapshot.durationMs;
		progress.phase = 'idle';
		return snapshot;
	} catch (error) {
		progress.error = String(error?.message || error);
		progress.phase = 'failed';
		throw error;
	} finally {
		progress.running = false;
		progress.finishedAt = new Date().toISOString();
		progress.current = '';
	}
}