Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(node-operator): run inference on a real GPU device and ship a CUDA image for it
nirholasb2030a4188c0
diff --git a/packages/node-operator/.dockerignore b/packages/node-operator/.dockerignorenew file mode 100644index 000000000..0ea13826d--- /dev/null+++ b/packages/node-operator/.dockerignore@@ -0,0 +1,10 @@+# Never bake an operator's identity, weights cache, or host node_modules into+# an image: the keypair would ship to anyone who pulls it, and the other two+# are rebuilt or re-downloaded inside the container anyway.+node-identity.json+operator.config.json+models/+node_modules/+tests/+scripts/+README.mddiff --git a/packages/node-operator/.gitignore b/packages/node-operator/.gitignorenew file mode 100644index 000000000..0dfcaab7e--- /dev/null+++ b/packages/node-operator/.gitignore@@ -0,0 +1,9 @@+# The node's ed25519 secret key. This IS the node identity, the payout+# address, and the signing key for every result receipt. It must never be+# committed; `src/index.js` writes it 0600 on first run.+node-identity.json+operator.config.json++# Downloaded model weights (tens of MB) and installed dependencies.+models/+node_modules/diff --git a/packages/node-operator/Dockerfile b/packages/node-operator/Dockerfileindex d22bbed44..0a4c6a471 100644--- a/packages/node-operator/Dockerfile+++ b/packages/node-operator/Dockerfile@@ -1,17 +1,24 @@# syntax=docker/dockerfile:1-# three.ws node operator client.+# three.ws node operator client, CPU image.#-# CPU builds work out of the box. GPU builds layer the NVIDIA Container-# Toolkit on top: `docker run --gpus all ...` with the same image; the-# @xenova/transformers ONNX runtime picks up CUDA execution providers when-# the drivers are present, and stays on CPU otherwise. No separate image.+# This image runs the proof workload on the CPU execution provider, which is+# what the vast majority of operators start on: no drivers, no toolkit, one+# `docker run` and the node is registered and polling.+#+# For an NVIDIA GPU, build Dockerfile.gpu instead. It is a separate image on+# purpose: onnxruntime's CUDA execution provider is a shared library that+# dynamically links the CUDA 12 runtime (libcublasLt.so.12 and friends), and+# those libraries are not in this base image. Running this image with+# `--gpus all` therefore does NOT get you GPU inference; it silently stays on+# the CPU. The client makes that visible either way: it logs the device each+# job actually ran on, and `DEVICE=cuda` fails loudly instead of downgrading.FROM node:22-bookworm-slimWORKDIR /app-# Install dependencies first (layer cached unless package.json changes).-COPY package.json ./-RUN npm install --omit=dev+# Install dependencies first (layer cached unless the manifest changes).+COPY package.json package-lock.json ./+RUN npm ci --omit=dev# Copy the client source.COPY src ./src@@ -22,6 +29,7 @@ ENV NODE_ENV=production \PLATFORM_URL=https://three.ws \MODEL=Xenova/all-MiniLM-L6-v2 \CAPABILITY=text-embedding \+ DEVICE=cpu \POLL_INTERVAL_MS=5000 \MAX_CONCURRENCY=1 \IDENTITY_PATH=/app/node-identity.json \@@ -29,6 +37,10 @@ ENV NODE_ENV=production \VOLUME ["/app/models"]+# The weights cache and the identity file are written at runtime by the+# unprivileged user, so the mount point has to belong to it before the drop.+RUN mkdir -p /app/models && chown -R node:node /app+# Run as the node user the base image ships: the operator's keypair and# downloaded weights never need root.USER nodediff --git a/packages/node-operator/Dockerfile.gpu b/packages/node-operator/Dockerfile.gpunew file mode 100644index 000000000..5d61bfbc7--- /dev/null+++ b/packages/node-operator/Dockerfile.gpu@@ -0,0 +1,59 @@+# syntax=docker/dockerfile:1+# three.ws node operator client, NVIDIA GPU image.+#+# docker build -f Dockerfile.gpu -t threews-node:gpu .+# docker run --gpus all -v threews-models:/app/models threews-node:gpu+#+# What makes this image different from the CPU one is the base layer, not the+# client code. onnxruntime-node ships libonnxruntime_providers_cuda.so, but+# that library dynamically links the CUDA 12 runtime and cuDNN 9; on a base+# image without them the loader fails with+# "libcublasLt.so.12: cannot open shared object file". The CUDA runtime image+# below supplies exactly those libraries, and the NVIDIA Container Toolkit+# (`--gpus all`) supplies the driver from the host.+#+# Requirements on the host: an NVIDIA driver new enough for CUDA 12 (>= 525)+# and the NVIDIA Container Toolkit installed. Without `--gpus all` this image+# runs, but DEVICE=cuda fails at startup rather than pretending.+FROM nvidia/cuda:12.6.3-cudnn-runtime-ubuntu24.04++# Node 22 from NodeSource: the Ubuntu archive's nodejs is too old for the+# client's `node >= 20` engine and its native ESM usage.+RUN apt-get update \+ && apt-get install -y --no-install-recommends ca-certificates curl gnupg \+ && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \+ && apt-get install -y --no-install-recommends nodejs \+ && apt-get purge -y --auto-remove gnupg \+ && rm -rf /var/lib/apt/lists/*++WORKDIR /app++COPY package.json package-lock.json ./+RUN npm ci --omit=dev++COPY src ./src++# DEVICE=cuda is deliberate here rather than `auto`: an operator who built the+# GPU image wants to know immediately if the GPU is not reachable, not to+# discover a week later that every job ran on the CPU.+ENV NODE_ENV=production \+ PLATFORM_URL=https://three.ws \+ MODEL=Xenova/all-MiniLM-L6-v2 \+ CAPABILITY=text-embedding \+ DEVICE=cuda \+ POLL_INTERVAL_MS=5000 \+ MAX_CONCURRENCY=1 \+ IDENTITY_PATH=/app/node-identity.json \+ HF_HOME=/app/models++VOLUME ["/app/models"]++# The CUDA base image has no unprivileged user; create one so the keypair and+# the weights cache are not written as root.+RUN useradd --create-home --shell /usr/sbin/nologin operator \+ && mkdir -p /app/models \+ && chown -R operator:operator /app++USER operator++CMD ["node", "src/index.js"]diff --git a/packages/node-operator/src/config.js b/packages/node-operator/src/config.jsindex 5a7e4ced1..409491101 100644--- a/packages/node-operator/src/config.js+++ b/packages/node-operator/src/config.js@@ -10,6 +10,8 @@import { readFileSync, existsSync } from 'node:fs';import { join } from 'node:path';+import { DEVICES } from './inference.js';+const DEFAULTS = {platformUrl: 'https://three.ws',model: 'Xenova/all-MiniLM-L6-v2',@@ -18,6 +20,7 @@ const DEFAULTS = {maxConcurrency: 1,identityPath: 'node-identity.json',jobTimeoutMs: 120_000,+ device: 'auto',};/**@@ -55,11 +58,18 @@ export function loadConfig({ env = process.env, cwd = process.cwd() } = {}) {identityPath: pick('IDENTITY_PATH', 'identityPath', DEFAULTS.identityPath),secretKey: env.OPERATOR_SECRET_KEY || file.secretKey || null,label: pick('NODE_LABEL', 'label', null),+ device: String(pick('DEVICE', 'device', DEFAULTS.device)).toLowerCase(),+ // Left null on purpose: the runtime picks the precision that matches the+ // chosen device (q8 on CPU, fp32 on GPU). Only override it deliberately.+ dtype: pick('DTYPE', 'dtype', null),};if (!/^https?:\/\//.test(cfg.platformUrl)) {throw new Error(`platformUrl must be an absolute http(s) URL, got: ${cfg.platformUrl}`);}+ if (!DEVICES.includes(cfg.device)) {+ throw new Error(`DEVICE must be one of ${DEVICES.join(', ')}, got: ${cfg.device}`);+ }if (!Number.isInteger(cfg.maxConcurrency) || cfg.maxConcurrency < 1) {throw new Error('maxConcurrency must be a positive integer');}diff --git a/packages/node-operator/src/index.js b/packages/node-operator/src/index.jsindex 9d704c956..bd41336a4 100644--- a/packages/node-operator/src/index.js+++ b/packages/node-operator/src/index.js@@ -36,12 +36,12 @@ async function main() {}console.log(`[node] identity ${identity.publicKey} (${source === 'generated' ? `generated, saved to ${identityPath}` : `from ${source}`})`);- console.log(`[node] platform ${cfg.platformUrl} · capability ${cfg.capability} · model ${cfg.model}`);+ console.log(`[node] platform ${cfg.platformUrl} · capability ${cfg.capability} · model ${cfg.model} · device ${cfg.device}`);if (args.has('--self-test')) {console.log('[node] running proof workload self-test (first run downloads the model)...');- const result = await selfTest({ cacheDir: MODELS_DIR });- console.log(`[node] self-test ${result.ok ? 'OK' : 'FAILED'}: ${result.model}, ${result.dimensions} dims, ${result.elapsedMs}ms`);+ const result = await selfTest({ cacheDir: MODELS_DIR, device: cfg.device, dtype: cfg.dtype });+ console.log(`[node] self-test ${result.ok ? 'OK' : 'FAILED'}: ${result.model}, ${result.dimensions} dims, ran on ${result.device} (${result.dtype}) in ${result.elapsedMs}ms`);process.exit(result.ok ? 0 : 1);}@@ -62,6 +62,8 @@ async function main() {maxConcurrency: cfg.maxConcurrency,jobTimeoutMs: cfg.jobTimeoutMs,cacheDir: MODELS_DIR,+ device: cfg.device,+ dtype: cfg.dtype,});const shutdown = (signal) => {diff --git a/packages/node-operator/src/inference.js b/packages/node-operator/src/inference.jsindex fb2dcc94e..73cfa4e2b 100644--- a/packages/node-operator/src/inference.js+++ b/packages/node-operator/src/inference.js@@ -1,8 +1,8 @@/*** The built-in proof workload: real, local text-embedding inference with the- * open Xenova/all-MiniLM-L6-v2 model (Apache-2.0, ~90MB ONNX, quantized for- * CPU). No API keys, no network calls at inference time: the model downloads- * from HuggingFace once and is cached in ./models thereafter.+ * open Xenova/all-MiniLM-L6-v2 model (Apache-2.0, ~23MB quantized ONNX). No+ * API keys, no network calls at inference time: the model downloads from the+ * HuggingFace hub once and is cached in ./models thereafter.** Why embeddings and not a generative model as the proof workload: it is the* smallest model that still exercises the full GPU/CPU tensor path a real@@ -11,46 +11,120 @@* deterministic, verifiable numeric output. Nodes advertise this capability* on registration as proof they can execute the runtime; richer models plug* into the same runJob() interface.+ *+ * Device selection is real, not decorative. @huggingface/transformers v4+ * bundles onnxruntime-node 1.24, which ships libonnxruntime_providers_cuda.so+ * for linux/x64, so `DEVICE=cuda` genuinely runs the forward pass on the GPU.+ * The predecessor library (@xenova/transformers v2) hardcoded+ * `executionProviders = ['cpu', 'wasm']` and shipped no CUDA provider at all,+ * which made the old "it picks up CUDA when the drivers are present" claim+ * false on every host.*/-import { createRequire } from 'node:module';+import { existsSync } from 'node:fs';-const require = createRequire(import.meta.url);+/** Devices the operator may request. 'auto' probes GPU first, then CPU. */+export const DEVICES = ['auto', 'cpu', 'cuda', 'webgpu', 'dml', 'coreml'];let _pipeline = null;-let _pipelineModel = null;+let _pipelineKey = null;+let _resolved = null;++/**+ * Is an NVIDIA driver actually attached to this host?+ *+ * The NVIDIA Container Toolkit (`docker run --gpus all`) mounts the driver+ * device nodes into the container, and a bare-metal host with the driver+ * loaded exposes /proc/driver/nvidia/version. Checking that before reaching+ * for CUDA is what keeps `DEVICE=auto` cheap: the fp32 graph is a 90MB+ * download and the q8 CPU graph is 23MB, so blindly trying CUDA first would+ * cost every CPU-only operator 90MB of weights they can never execute.+ */+export function hasNvidiaDriver({ platform = process.platform, arch = process.arch, exists = existsSync } = {}) {+ if (platform !== 'linux' || arch !== 'x64') return false;+ return exists('/dev/nvidiactl') || exists('/dev/nvidia0') || exists('/proc/driver/nvidia/version');+}++/**+ * Candidate execution devices, most capable first, for a requested setting.+ *+ * An explicit device is honored exactly: asking for 'cuda' on a host with no+ * GPU must fail loudly rather than quietly bill the operator's electricity+ * for a CPU run they did not ask for. 'auto' is the one setting that falls+ * back, and it says so in the log line.+ */+export function deviceCandidates(requested = 'auto', { platform = process.platform, arch = process.arch, gpu } = {}) {+ if (requested && requested !== 'auto') return [requested];+ const nvidia = gpu === undefined ? hasNvidiaDriver({ platform, arch }) : gpu;+ if (platform === 'linux' && arch === 'x64') return nvidia ? ['cuda', 'cpu'] : ['cpu'];+ if (platform === 'darwin') return ['coreml', 'cpu'];+ if (platform === 'win32') return ['dml', 'cpu'];+ return ['cpu'];+}++/**+ * Weight precision for a device. CUDA loads the full fp32 graph (the GPU has+ * the memory and int8 kernels buy nothing there); CPU loads the q8 quantized+ * graph, which is what the 23MB cached download is.+ */+export function defaultDtype(device) {+ return device === 'cpu' ? 'q8' : 'fp32';+}/*** Load (and cache) the feature-extraction pipeline for a model id. The first* call downloads the model weights; later calls return the warm pipeline.* `cacheDir` pins where weights land so Docker volumes can persist them.+ *+ * @returns {Promise<{extractor: Function, device: string, dtype: string}>}*/-export async function loadModel(model, { cacheDir } = {}) {- if (_pipeline && _pipelineModel === model) return _pipeline;- const { pipeline, env } = await import('@xenova/transformers');+export async function loadModel(model, { cacheDir, device = 'auto', dtype, log = console } = {}) {+ const key = `${model}|${device}|${dtype || ''}`;+ if (_pipeline && _pipelineKey === key) return { extractor: _pipeline, ..._resolved };+ const { pipeline, env } = await import('@huggingface/transformers');if (cacheDir) env.cacheDir = cacheDir;// The model hub is the only remote allowed, and only on first download.env.allowLocalModels = true;- _pipeline = await pipeline('feature-extraction', model, { quantized: true });- _pipelineModel = model;- return _pipeline;++ const candidates = deviceCandidates(device);+ const failures = [];+ for (const candidate of candidates) {+ const precision = dtype || defaultDtype(candidate);+ try {+ _pipeline = await pipeline('feature-extraction', model, { device: candidate, dtype: precision });+ _pipelineKey = key;+ _resolved = { device: candidate, dtype: precision };+ if (failures.length) {+ log.warn?.(`[node] ${failures.length} device(s) unavailable, running on ${candidate}: ${failures.join(' | ')}`);+ }+ return { extractor: _pipeline, ..._resolved };+ } catch (err) {+ failures.push(`${candidate}: ${err.message}`);+ }+ }+ throw new Error(`could not load ${model} on any of [${candidates.join(', ')}] -> ${failures.join(' | ')}`);}/*** Execute one inference job.*+ * The returned output deliberately carries no device or dtype field: two+ * honest nodes running the same job must produce the same output shape, and+ * the result receipt hashes this object. Runtime details belong in the+ * operator's own logs, not in a signed result other nodes have to match.+ ** @param {object} job the claimed job: { id, model, input }- * @param {object} [opts] { cacheDir, now } - now is injectable for tests- * @returns {Promise<{output: object, startedAt: number, finishedAt: number}>}+ * @param {object} [opts] { cacheDir, device, dtype, now } - now is injectable for tests+ * @returns {Promise<{output: object, startedAt: number, finishedAt: number, device: string}>}*/-export async function runJob(job, { cacheDir, now = () => Date.now() } = {}) {+export async function runJob(job, { cacheDir, device = 'auto', dtype, log = console, now = () => Date.now() } = {}) {const prompt = typeof job.input === 'string' ? job.input : job.input?.text;if (typeof prompt !== 'string' || prompt.length === 0) {throw new Error(`job ${job.id}: input.text must be a non-empty string`);}const model = job.model || 'Xenova/all-MiniLM-L6-v2';const startedAt = now();- const extractor = await loadModel(model, { cacheDir });+ const { extractor, device: resolvedDevice, dtype: resolvedDtype } = await loadModel(model, { cacheDir, device, dtype, log });const tensor = await extractor(prompt, { pooling: 'mean', normalize: true });const finishedAt = now();return {@@ -62,22 +136,27 @@ export async function runJob(job, { cacheDir, now = () => Date.now() } = {}) {},startedAt,finishedAt,+ device: resolvedDevice,+ dtype: resolvedDtype,};}/*** A self-check the operator can run before registering: executes the proof- * workload on a fixed input and returns the output summary. `node- * src/index.js --self-test` calls this so a fresh operator can prove their- * host runs the runtime before they ever poll for jobs.+ * workload on a fixed input and returns the output summary, including the+ * device that actually ran it. `node src/index.js --self-test` calls this so+ * a fresh operator can prove their host runs the runtime (and see whether+ * their GPU was really picked up) before they ever poll for jobs.*/-export async function selfTest({ cacheDir } = {}) {+export async function selfTest({ cacheDir, device = 'auto', dtype, log = console } = {}) {const probe = { id: 'self-test', model: undefined, input: { text: 'three.ws node operator self-test' } };- const { output, startedAt, finishedAt } = await runJob(probe, { cacheDir });+ const { output, startedAt, finishedAt, device: resolvedDevice, dtype: resolvedDtype } = await runJob(probe, { cacheDir, device, dtype, log });return {ok: Array.isArray(output.embedding) && output.embedding.length === output.dimensions,model: output.model,dimensions: output.dimensions,+ device: resolvedDevice,+ dtype: resolvedDtype,elapsedMs: finishedAt - startedAt,};}diff --git a/packages/node-operator/src/loop.js b/packages/node-operator/src/loop.jsindex 21ef8598e..781d888b4 100644--- a/packages/node-operator/src/loop.js+++ b/packages/node-operator/src/loop.js@@ -20,6 +20,8 @@ export function createJobLoop({pollIntervalMs = 5000,maxConcurrency = 1,cacheDir,+ device = 'auto',+ dtype,jobTimeoutMs = 120_000,log = console,runJobImpl = runJob,@@ -51,8 +53,8 @@ export function createJobLoop({async function executeJob(job) {const startedAt = Date.now();try {- const { output, startedAt: t0, finishedAt: t1 } = await withTimeout(- runJobImpl(job, { cacheDir }),+ const { output, startedAt: t0, finishedAt: t1, device: ranOn } = await withTimeout(+ runJobImpl(job, { cacheDir, device, dtype, log }),jobTimeoutMs,`job ${job.id} exceeded ${jobTimeoutMs}ms`,);@@ -66,7 +68,7 @@ export function createJobLoop({});const res = await client.submitResult(job.id, { output, startedAt: t0, finishedAt: t1, receipt });jobsDone.completed++;- log.log(`[node] job ${job.id} complete (${t1 - t0}ms inference, verified=${res?.verified ?? 'unknown'})`);+ log.log(`[node] job ${job.id} complete (${t1 - t0}ms inference on ${ranOn ?? 'unknown device'}, verified=${res?.verified ?? 'unknown'})`);} catch (err) {jobsDone.failed++;log.warn(`[node] job ${job.id} failed: ${err.message}`);