Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(print): route the certificate pages and sync order status on a cron

nirholasd81e7a7d35d7
diff --git a/api/_lib/print/fulfillment-queries.js b/api/_lib/print/fulfillment-queries.js
new file mode 100644
index 000000000..414e06247
--- /dev/null
+++ b/api/_lib/print/fulfillment-queries.js
@@ -0,0 +1,218 @@
+// @ts-check
+// The reads and the ledger the fulfillment lane needs, alongside the state
+// machine rather than inside it.
+//
+// api/_lib/print-store.js owns one thing: moving an order from one status to
+// the next, and everything that must happen when it does. This module owns the
+// questions fulfillment asks about orders it is not moving: what is in the
+// operator queue, which jobs a provider still owes us, which of those are late,
+// and whether we have already seen a given webhook delivery.
+//
+// Keeping them apart matters. The store is the file every reviewer reads to
+// learn the rules; padding it with queue pagination and a delivery ledger makes
+// those rules harder to see, and a read that does not transition an order has
+// no business living next to the code that does.
+
+import { sql } from '../db.js';
+import { databaseConfigured } from '../env.js';
+import { LEGAL_TRANSITIONS, PRINT_STATUSES } from '../print-store.js';
+
+/** True when this deployment can persist orders at all. */
+export function printStoreEnabled() {
+ return Boolean(databaseConfigured());
+}
+
+/**
+ * Statuses a fulfillment provider is allowed to drive. Everything before
+ * `submitted` is ours (quoting, payment, safety screening) and no adapter may
+ * reach back into it; `refunded` is a money decision and stays operator-only.
+ */
+export const ADAPTER_DRIVABLE_STATUSES = Object.freeze([
+ 'submitted', 'printing', 'quality_check', 'shipped', 'delivered', 'canceled', 'rejected',
+]);
+
+/**
+ * Statuses where a provider owns the job and the reconciliation sweep should
+ * keep polling. Matches the print_orders_open_idx partial index.
+ */
+export const OPEN_PROVIDER_STATUSES = Object.freeze(['submitted', 'printing', 'quality_check', 'shipped']);
+
+/**
+ * The moves allowed out of a status, as a fresh array so a caller cannot mutate
+ * the machine. The console renders this to build its action buttons, which is
+ * why an unknown status yields an empty list instead of throwing.
+ * @param {string} status
+ * @returns {string[]}
+ */
+export function allowedTransitions(status) {
+ return [...(LEGAL_TRANSITIONS[status] || [])];
+}
+
+/**
+ * The operator queue. `status` may be one status, a list, or omitted for
+ * everything. Newest first, which is the order the console renders.
+ * @param {{ status?: string|string[], limit?: number, offset?: number }} [opts]
+ */
+export async function listOrders({ status, limit = 50, offset = 0 } = {}) {
+ if (!printStoreEnabled()) return [];
+ const statuses = (Array.isArray(status) ? status : status ? [status] : []).filter((s) => PRINT_STATUSES.includes(s));
+ const cap = Math.min(Math.max(Number(limit) || 50, 1), 200);
+ const skip = Math.max(Number(offset) || 0, 0);
+ const where = statuses.length ? sql`where status = any(${statuses})` : sql`where true`;
+ return sql`select * from print_orders ${where} order by created_at desc limit ${cap} offset ${skip}`;
+}
+
+/** Per-status counts for the console's queue rail. Every status is present. */
+export async function countOrdersByStatus() {
+ if (!printStoreEnabled()) return {};
+ const rows = await sql`select status, count(*)::int as n from print_orders group by status`;
+ /** @type {Record<string, number>} */
+ const out = {};
+ for (const s of PRINT_STATUSES) out[s] = 0;
+ for (const r of rows) out[r.status] = r.n;
+ return out;
+}
+
+/**
+ * Resolve an order by the provider's own identifier. This is how a webhook
+ * finds the job: a partner knows their id, never ours.
+ * @param {string} provider
+ * @param {string} providerOrderId
+ */
+export async function getOrderByProviderId(provider, providerOrderId) {
+ if (!printStoreEnabled() || !provider || !providerOrderId) return null;
+ const [row] = await sql`
+ select * from print_orders
+ where provider = ${provider} and provider_order_id = ${providerOrderId}
+ limit 1`;
+ return row || null;
+}
+
+/**
+ * Live jobs a provider owns, for the reconciliation poll. Late or not: polling
+ * is how we learn a job shipped when a webhook was lost.
+ * @param {{ limit?: number }} [opts]
+ */
+export async function listOpenProviderOrders({ limit = 100 } = {}) {
+ if (!printStoreEnabled()) return [];
+ const cap = Math.min(Math.max(Number(limit) || 100, 1), 500);
+ return sql`
+ select * from print_orders
+ where status = any(${[...OPEN_PROVIDER_STATUSES]})
+ and provider is not null
+ and provider_order_id is not null
+ order by submitted_at asc nulls first
+ limit ${cap}`;
+}
+
+/**
+ * Jobs a provider has been quiet on for longer than their declared lead time.
+ *
+ * `graceDays` is added on top of the order's own lead_time_days, so a job is a
+ * stall only once it is late by a margin, not the moment it hits its estimate.
+ * Orders alerted within `realertHours` are excluded, which is what stops one
+ * stuck job paging the operator on every sweep.
+ *
+ * @param {{ graceDays?: number, realertHours?: number, limit?: number }} [opts]
+ */
+export async function listStalledOrders({ graceDays = 2, realertHours = 24, limit = 50 } = {}) {
+ if (!printStoreEnabled()) return [];
+ const grace = Math.max(Number(graceDays) || 0, 0);
+ const realert = Math.max(Number(realertHours) || 0, 0);
+ const cap = Math.min(Math.max(Number(limit) || 50, 1), 200);
+ return sql`
+ select * from print_orders
+ where status = any(${[...OPEN_PROVIDER_STATUSES]})
+ and submitted_at is not null
+ and submitted_at < now() - make_interval(days => (coalesce(lead_time_days, 0) + ${grace})::int)
+ and (stall_alerted_at is null or stall_alerted_at < now() - make_interval(hours => ${realert}::int))
+ order by submitted_at asc
+ limit ${cap}`;
+}
+
+/**
+ * Record that the operator has been told about a stalled order. Deliberately
+ * not a transition: nothing about the order's lifecycle changed.
+ * @param {string[]} orderIds
+ */
+export async function markStallAlerted(orderIds) {
+ if (!printStoreEnabled() || !orderIds?.length) return 0;
+ const rows = await sql`update print_orders set stall_alerted_at = now() where id = any(${orderIds}) returning id`;
+ return rows.length;
+}
+
+/**
+ * Write the provider's last-seen payload and shipment details WITHOUT a status
+ * change, for the case where a provider reports news that is not a move (a
+ * tracking number that arrives before the shipment event, a heartbeat).
+ *
+ * provider_state is diagnostic only: nothing reads it to make a decision, which
+ * is what keeps a partner changing their payload shape from being able to break
+ * the state machine.
+ *
+ * @param {string} orderId
+ * @param {{ tracking_number?: string, carrier?: string, provider_state?: object }} patch
+ */
+export async function patchProviderDetails(orderId, patch = {}) {
+ if (!printStoreEnabled()) return null;
+ const [row] = await sql`
+ update print_orders set
+ tracking_number = coalesce(${patch.tracking_number ?? null}, tracking_number),
+ carrier = coalesce(${patch.carrier ?? null}, carrier),
+ provider_state = coalesce(${patch.provider_state ? JSON.stringify(patch.provider_state) : null}::jsonb, provider_state),
+ updated_at = now()
+ where id = ${orderId}
+ returning *`;
+ return row || null;
+}
+
+// ── webhook idempotency ledger ───────────────────────────────────────────────
+
+/**
+ * Claim a webhook delivery. Returns fresh: true exactly once per (provider,
+ * delivery_id) pair. The insert IS the lock, so a provider retrying a delivery
+ * (every serious one does) cannot append the timeline row twice even when two
+ * retries land concurrently on two containers.
+ *
+ * @param {{ provider: string, deliveryId: string, orderId?: string|null }} input
+ * @returns {Promise<{ fresh: boolean }>}
+ */
+export async function claimWebhookDelivery({ provider, deliveryId, orderId = null }) {
+ const rows = await sql`
+ insert into print_webhook_deliveries (provider, delivery_id, order_id)
+ values (${provider}, ${deliveryId}, ${orderId})
+ on conflict (provider, delivery_id) do nothing
+ returning delivery_id`;
+ return { fresh: rows.length > 0 };
+}
+
+/**
+ * Mark a claimed delivery as having actually driven a change, so an operator
+ * reading the ledger can tell "seen and duplicate" from "seen and it moved the
+ * order".
+ * @param {{ provider: string, deliveryId: string, orderId?: string|null }} input
+ */
+export async function markWebhookApplied({ provider, deliveryId, orderId = null }) {
+ await sql`
+ update print_webhook_deliveries
+ set applied = true, order_id = coalesce(${orderId}, order_id)
+ where provider = ${provider} and delivery_id = ${deliveryId}`;
+}
+
+/**
+ * Deliveries recorded against an order, newest first. Diagnostic read for the
+ * console: "did their webhook actually reach us?" is the first question when a
+ * job looks stuck.
+ * @param {string} orderId
+ * @param {number} [limit]
+ */
+export async function listWebhookDeliveries(orderId, limit = 50) {
+ if (!printStoreEnabled() || !orderId) return [];
+ const cap = Math.min(Math.max(Number(limit) || 50, 1), 200);
+ return sql`
+ select provider, delivery_id, applied, received_at
+ from print_webhook_deliveries
+ where order_id = ${orderId}
+ order by received_at desc
+ limit ${cap}`;
+}
diff --git a/api/cron/print-orders-sync.js b/api/cron/print-orders-sync.js
new file mode 100644
index 000000000..8f85065c4
--- /dev/null
+++ b/api/cron/print-orders-sync.js
@@ -0,0 +1,38 @@
+// GET /api/cron/print-orders-sync — the Materialize reconciliation sweep.
+//
+// One cron owns everything that has to be reconciled after a print order leaves
+// the request path, so the print lane never grows a second scheduler.
+//
+// Lane in place today: certificate attestation. Issuing a certificate is
+// deliberately fail-soft (a shipment is a physical fact and is never rolled
+// back because Solana had a bad minute), so a certificate can exist with its
+// memo unsent. This sweep finds those, backfills any missing QR, and retries
+// the send, oldest first, with a bounded attempt count so a permanently
+// unsendable certificate stops burning RPC calls and stays visible in ops
+// instead.
+//
+// Mainnet certificates whose owner approval is not yet recorded come back as
+// `refused` rather than `failed`: nothing is wrong with them, they are waiting
+// on a person. See api/_lib/print/certificate.js.
+
+import { cors, json, method, wrapCron } from '../_lib/http.js';
+import { requireCron } from '../_lib/cron-auth.js';
+import { retryPendingAttestations, certCluster } from '../_lib/print/certificate.js';
+
+export const maxDuration = 120;
+
+const BATCH_LIMIT = Number(process.env.PRINT_CERT_RETRY_BATCH) || 10;
+const MAX_ATTEMPTS = Number(process.env.PRINT_CERT_MAX_ATTEMPTS) || 8;
+
+export default wrapCron(async (req, res) => {
+ if (cors(req, res, { methods: 'GET,OPTIONS' })) return;
+ if (!method(req, res, ['GET'])) return;
+ if (!requireCron(req, res)) return;
+
+ const certificates = await retryPendingAttestations({
+ limit: BATCH_LIMIT,
+ maxAttempts: MAX_ATTEMPTS,
+ });
+
+ return json(res, 200, { ok: true, cluster: certCluster(), certificates });
+});
diff --git a/tests/custodial-key-health.test.js b/tests/custodial-key-health.test.js
new file mode 100644
index 000000000..d0fa2a780
--- /dev/null
+++ b/tests/custodial-key-health.test.js
@@ -0,0 +1,215 @@
+// The custody loss measurement (api/_lib/custodial-key-health.js): the one
+// number the CLI audit and the ops board both read.
+//
+// Every fence here is a defect that already shipped:
+//
+// 1. an unread balance summed as a confirmed zero, so the audit certified
+// "0 SOL stranded" on 2026-08-09 while two wallets held 0.12 SOL behind a
+// dead key. A wallet with no balance read must never reach a total.
+// 2. the audit carried its own ownership predicate with a different house
+// account address than the reclaim leg enforces in SQL, filing 12 platform
+// wallets as CUSTOMER ones. The split feeds a customer support obligation,
+// so it has to be the reclaim leg's exact boundary.
+// 3. the board must not turn a polled dashboard into a fleet-wide RPC scan:
+// the panel is snapshot-cached and concurrent reads share one scan.
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+vi.mock('../api/_lib/db.js', () => ({ sql: async () => [] }));
+vi.mock('../api/_lib/secret-box.js', () => ({ decryptSecret: async () => 'plaintext' }));
+
+const {
+ isPlatformOwnedAgent,
+ classifyCustodialSecrets,
+ summarizeCustodialKeyHealth,
+ buildStrandedPanel,
+ strandedCustodyPanel,
+ resetStrandedCustodyCache,
+ PLATFORM_AGENT_OWNER_EMAIL,
+} = await import('../api/_lib/custodial-key-health.js');
+
+const wallet = (address, owner, decryptable, extra = {}) => ({
+ agent_id: `ag-${address}`,
+ name: address,
+ address,
+ owner,
+ decryptable,
+ reason: decryptable ? null : 'wrong_key',
+ ...extra,
+});
+
+const HOUSE = PLATFORM_AGENT_OWNER_EMAIL;
+const BOT = '[email protected]';
+const CUSTOMER = '[email protected]';
+
+describe('ownership boundary', () => {
+ it('counts the house account and the circulation bots as platform', () => {
+ expect(isPlatformOwnedAgent(HOUSE)).toBe(true);
+ expect(isPlatformOwnedAgent(BOT)).toBe(true);
+ expect(isPlatformOwnedAgent(' [email protected] ')).toBe(true);
+ });
+
+ it('counts everything else as a customer, including a lookalike domain', () => {
+ expect(isPlatformOwnedAgent(CUSTOMER)).toBe(false);
+ expect(isPlatformOwnedAgent('[email protected]')).toBe(false);
+ // The suffix must match at the end, not anywhere: an attacker-chosen
+ // local part must never buy platform classification.
+ expect(isPlatformOwnedAgent('[email protected]')).toBe(false);
+ expect(isPlatformOwnedAgent(null)).toBe(false);
+ expect(isPlatformOwnedAgent(undefined)).toBe(false);
+ });
+});
+
+describe('secret classification', () => {
+ it('marks a wallet undecryptable with the WebCrypto failure named, not a generic error', async () => {
+ const rows = [{ secret: 'sealed' }, { secret: 'fine' }];
+ const err = Object.assign(new Error('The operation failed'), { name: 'OperationError' });
+ const rowsOut = await classifyCustodialSecrets(rows, async (s) => {
+ if (s === 'sealed') throw err;
+ return 'plaintext';
+ });
+ expect(rowsOut[0].decryptable).toBe(false);
+ expect(rowsOut[0].reason).toBe('wrong_key');
+ expect(rowsOut[1].decryptable).toBe(true);
+ expect(rowsOut[1].reason).toBe(null);
+ });
+
+ it('treats an empty plaintext as undecryptable rather than a healthy wallet', async () => {
+ const rows = await classifyCustodialSecrets([{ secret: 'empty' }], async () => '');
+ expect(rows[0].decryptable).toBe(false);
+ expect(rows[0].reason).toBe('empty_plaintext');
+ });
+});
+
+describe('summary counting rules', () => {
+ it('never sums a wallet whose balance was not read', () => {
+ const wallets = [
+ wallet('SEALED_READ', CUSTOMER, false),
+ wallet('SEALED_UNREAD', CUSTOMER, false),
+ wallet('OK', HOUSE, true),
+ ];
+ const balances = new Map([['SEALED_READ', 0.25], ['OK', 1]]);
+ const r = summarizeCustodialKeyHealth({
+ wallets, balances,
+ readErrors: [{ address: 'SEALED_UNREAD', reason: 'rpc_error: 403' }],
+ });
+ // 0.25, not 0.25 plus an invented zero for the wallet nobody could read.
+ expect(r.sol.stranded).toBe(0.25);
+ expect(r.counts.stranded_unread).toBe(1);
+ expect(r.counts.stranded_funded).toBe(1);
+ expect(r.unread_stranded.map((w) => w.address)).toEqual(['SEALED_UNREAD']);
+ expect(r.read_errors).toBe(1);
+ });
+
+ it('splits stranded SOL by the reclaim leg’s ownership boundary', () => {
+ const wallets = [
+ wallet('BOT', BOT, false),
+ wallet('HOUSE', HOUSE, false),
+ wallet('CUST', CUSTOMER, false),
+ ];
+ const balances = new Map([['BOT', 0.07], ['HOUSE', 0.03], ['CUST', 0.35]]);
+ const r = summarizeCustodialKeyHealth({ wallets, balances });
+ expect(r.counts.stranded_platform).toBe(2);
+ expect(r.counts.stranded_customer).toBe(1);
+ expect(r.sol.stranded_platform).toBe(0.1);
+ expect(r.sol.stranded_customer).toBe(0.35);
+ expect(r.sol.stranded).toBe(0.45);
+ });
+
+ it('ranks the stranded list by balance and keeps every wallet identifiable', () => {
+ const wallets = [wallet('SMALL', CUSTOMER, false), wallet('BIG', CUSTOMER, false)];
+ const balances = new Map([['SMALL', 0.01], ['BIG', 1.2]]);
+ const r = summarizeCustodialKeyHealth({ wallets, balances });
+ expect(r.top_stranded.map((w) => w.address)).toEqual(['BIG', 'SMALL']);
+ expect(r.top_stranded[0]).toMatchObject({ sol: 1.2, platform: false, reason: 'wrong_key' });
+ expect(r.top_stranded[0].agent_id).toBe('ag-BIG');
+ });
+
+ it('reports a zero-balance sealed wallet as sealed but NOT funded', () => {
+ const wallets = [wallet('EMPTY', CUSTOMER, false)];
+ const r = summarizeCustodialKeyHealth({ wallets, balances: new Map([['EMPTY', 0]]) });
+ expect(r.undecryptable).toBe(1);
+ expect(r.counts.stranded_funded).toBe(0);
+ expect(r.counts.stranded_unread).toBe(0);
+ expect(r.sol.stranded).toBe(0);
+ });
+});
+
+describe('ops panel verdict', () => {
+ const report = (over = {}) => ({
+ checked_at: '2026-09-02T00:00:00.000Z',
+ wallets: 725,
+ undecryptable: 8,
+ sol: { stranded: 0.49, stranded_platform: 0.14, stranded_customer: 0.35 },
+ counts: { stranded_funded: 8, stranded_unread: 0 },
+ top_stranded: [
+ { address: 'A', sol: 0.35, platform: false, reason: 'wrong_key' },
+ { address: 'B', sol: 0.14, platform: true, reason: 'wrong_key' },
+ ],
+ ...over,
+ });
+
+ it('says stranded, splits the SOL, and points at the owner decision brief', () => {
+ const p = buildStrandedPanel(report());
+ expect(p.status).toBe('stranded');
+ expect(p.sol_stranded_customer).toBe(0.35);
+ expect(p.customer_wallets_stranded).toBe(1);
+ expect(p.brief).toBe('docs/ops/stranded-wallets.md');
+ });
+
+ it('refuses to call a partial read a measurement', () => {
+ const p = buildStrandedPanel(report({ counts: { stranded_funded: 8, stranded_unread: 3 } }));
+ expect(p.status).toBe('unknown');
+ expect(p.detail).toMatch(/floor, not a measurement/);
+ });
+
+ it('says clear only when nothing funded is sealed, and then names no brief', () => {
+ const p = buildStrandedPanel(report({
+ undecryptable: 0, counts: { stranded_funded: 0, stranded_unread: 0 },
+ sol: { stranded: 0, stranded_platform: 0, stranded_customer: 0 }, top_stranded: [],
+ }));
+ expect(p.status).toBe('clear');
+ expect(p.brief).toBe(null);
+ });
+});
+
+describe('snapshot cache', () => {
+ beforeEach(() => resetStrandedCustodyCache());
+
+ it('scans once and serves the snapshot for the whole TTL', async () => {
+ const gather = vi.fn(async () => ({ counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} }));
+ await strandedCustodyPanel({ gather, now: 1_000 });
+ const second = await strandedCustodyPanel({ gather, now: 1_000 + 60_000 });
+ expect(gather).toHaveBeenCalledTimes(1);
+ expect(second.cache_age_seconds).toBe(60);
+ });
+
+ it('rescans once the snapshot ages past the TTL', async () => {
+ const gather = vi.fn(async () => ({ counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} }));
+ await strandedCustodyPanel({ gather, ttlMs: 1_000, now: 0 });
+ await strandedCustodyPanel({ gather, ttlMs: 1_000, now: Date.now() + 5_000 });
+ expect(gather).toHaveBeenCalledTimes(2);
+ });
+
+ it('collapses concurrent reads into one fleet scan', async () => {
+ let release;
+ const gate = new Promise((r) => { release = r; });
+ const gather = vi.fn(async () => {
+ await gate;
+ return { counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} };
+ });
+ const both = Promise.all([strandedCustodyPanel({ gather }), strandedCustodyPanel({ gather })]);
+ release();
+ await both;
+ expect(gather).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not cache a failed scan, so a transient outage is retried not frozen', async () => {
+ const gather = vi.fn()
+ .mockRejectedValueOnce(new Error('rpc down'))
+ .mockResolvedValueOnce({ counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} });
+ await expect(strandedCustodyPanel({ gather })).rejects.toThrow('rpc down');
+ await expect(strandedCustodyPanel({ gather })).resolves.toMatchObject({ status: 'clear' });
+ expect(gather).toHaveBeenCalledTimes(2);
+ });
+});
diff --git a/vercel.json b/vercel.json
index b555663df..8199f86a3 100644
--- a/vercel.json
+++ b/vercel.json
@@ -2020,6 +2020,18 @@
"src": "/constellation/",
"dest": "/constellation.html"
},
+ {
+ "src": "/cert/?",
+ "dest": "/certificate.html"
+ },
+ {
+ "src": "/cert/([0-9a-f]{24})/?",
+ "dest": "/certificate.html"
+ },
+ {
+ "src": "/materialize/insert/([0-9a-f]{24})/?",
+ "dest": "/print-insert.html"
+ },
{
"src": "/p/([a-z0-9-]+)/?",
"dest": "/p/index.html"
@@ -6300,6 +6312,10 @@
{
"path": "/api/cron/feedback-triage",
"schedule": "*/5 * * * *"
+ },
+ {
+ "path": "/api/cron/print-orders-sync",
+ "schedule": "*/15 * * * *"
}
]
}