Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
#!/usr/bin/env node
/**
* Pull every X Article an account has published, with what readers said back.
*
* A timeline scrape sees an Article as a title card. The X API v2 returns the
* whole thing when a post is read with the `article` tweet field: title,
* preview, cover, and the full plain text with its code blocks and links. This
* script walks the account's timeline (the API returns the last 3,200 posts),
* keeps the posts that carry an Article, and for each one collects the replies
* in its conversation (full-archive search) and the posts that quote it. Those
* two lists are the closest thing the account has to a reader survey: what
* people asked, what they pushed back on, what they wanted next.
*
* Usage:
* npm run x:articles:pull # every Article on @trythreews
* npm run x:articles:pull -- --no-replies # Articles and metrics only
* npm run x:articles:pull -- --since 2026-08-01
*
* Output: data/x-archive/<handle>-articles-<YYYY-MM-DD>.json. Snapshots are
* additive, like the rest of the archive (see data/x-archive/README.md).
*
* Credentials: X_API_KEY, X_API_SECRET, X_ACCESS_TOKEN, X_ACCESS_SECRET, the
* posting account's OAuth 1.0a user context. Read from the environment, then
* .env.local and .env, then the Cloud Run service through
* scripts/read-service-env.mjs. Read-only: nothing is posted.
*/
import { execFileSync } from 'node:child_process';
import { readFileSync } from 'node:fs';
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { xClientFromEnv } from '../api/_lib/x-content/publisher.js';
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const OUT_DIR = path.join(REPO_ROOT, 'data', 'x-archive');
const CREDENTIALS = ['X_API_KEY', 'X_API_SECRET', 'X_ACCESS_TOKEN', 'X_ACCESS_SECRET'];
const TIMELINE_CAP = 3200;
const args = process.argv.slice(2);
const flag = (name) => args.includes(`--${name}`);
const value = (name) => {
const i = args.indexOf(`--${name}`);
return i >= 0 ? args[i + 1] : null;
};
const SINCE = value('since');
const WITH_REPLIES = !flag('no-replies');
function loadEnvFiles() {
for (const file of ['.env.local', '.env']) {
let raw;
try {
raw = readFileSync(path.join(REPO_ROOT, file), 'utf8');
} catch {
continue;
}
for (const line of raw.split('\n')) {
const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
if (!m || process.env[m[1]]) continue;
process.env[m[1]] = m[2].trim().replace(/^(['"])(.*)\1$/, '$2');
}
}
}
function loadServiceCredentials() {
for (const name of CREDENTIALS) {
if (process.env[name]) continue;
const out = execFileSync('node', ['scripts/read-service-env.mjs', `^${name}$`, '--raw'], { cwd: REPO_ROOT, encoding: 'utf8' }).trim();
if (out) process.env[name] = out;
}
}
const TWEET_FIELDS = ['public_metrics', 'created_at', 'article', 'note_tweet', 'entities', 'referenced_tweets', 'conversation_id', 'author_id', 'in_reply_to_user_id'];
// The quotes endpoint can hand back a page it already served, so a post is
// kept once by id, and the walk stops after a full page brings nothing new.
const PAGE_SIZE = 100;
const READER_QUERY = {
'tweet.fields': ['public_metrics', 'created_at', 'note_tweet', 'referenced_tweets', 'author_id', 'in_reply_to_user_id'],
expansions: ['author_id'],
'user.fields': ['username', 'name', 'public_metrics', 'verified'],
max_results: PAGE_SIZE,
};
async function collect(paginator) {
const posts = [];
const seen = new Set();
let repeats = 0;
for await (const post of paginator) {
if (seen.has(post.id)) {
if (++repeats >= PAGE_SIZE) break;
continue;
}
repeats = 0;
seen.add(post.id);
posts.push(post);
}
const users = new Map((paginator.includes?.users || []).map((u) => [u.id, u]));
// A repost of a post that quotes the Article comes back from the quotes
// endpoint too. It carries no words of its own, so it is not a response.
const isRepost = (post) => (post.referenced_tweets || []).some((ref) => ref.type === 'retweeted');
return posts.filter((post) => !isRepost(post)).map((post) => {
const author = users.get(post.author_id);
return {
id: post.id,
url: author ? `https://x.com/${author.username}/status/${post.id}` : `https://x.com/i/status/${post.id}`,
created_at: post.created_at,
author: { id: post.author_id, username: author?.username ?? null, name: author?.name ?? null, followers: author?.public_metrics?.followers_count ?? null },
text: post.note_tweet?.text || post.text,
metrics: post.public_metrics,
};
});
}
// Full-archive search allows one request a second and a small budget per 15
// minutes. On a 429 wait until the reset X reports, then run the step again.
async function withRateLimit(label, step) {
for (;;) {
try {
return await step();
} catch (err) {
if (err?.code !== 429) throw err;
const reset = Number(err.rateLimit?.reset || 0) * 1000;
const waitMs = Math.max(15_000, reset - Date.now() + 2_000);
console.warn(` rate limited on ${label}; waiting ${Math.round(waitMs / 1000)}s`);
await new Promise((resolve) => setTimeout(resolve, waitMs));
}
}
}
async function readerResponse(client, article, selfId) {
const replies = await withRateLimit('replies', async () => collect(await client.searchAll(`conversation_id:${article.id} -from:${selfId}`, { ...READER_QUERY, start_time: article.created_at })));
const quotes = await withRateLimit('quotes', async () => collect(await client.quotes(article.id, READER_QUERY)));
return { replies, quotes: quotes.filter((q) => q.author.id !== selfId) };
}
function describe(post, handle) {
const a = post.article;
return {
id: post.id,
url: `https://x.com/${handle}/status/${post.id}`,
created_at: post.created_at,
title: a.title?.trim() || null,
preview: a.preview_text || null,
cover_media: a.cover_media || null,
characters: (a.plain_text || '').length,
metrics: post.public_metrics,
body: a.plain_text || null,
entities: a.entities || null,
};
}
async function main() {
loadEnvFiles();
loadServiceCredentials();
const client = await xClientFromEnv();
if (!client) throw new Error(`missing X credentials: set ${CREDENTIALS.join(', ')} or authenticate gcloud so the Cloud Run copies can be read`);
const { data: me } = await client.me();
const timeline = await client.userTimeline(me.id, { 'tweet.fields': TWEET_FIELDS, max_results: 100, exclude: ['retweets'], ...(SINCE ? { start_time: new Date(SINCE).toISOString() } : {}) });
const posts = [];
for await (const post of timeline) {
posts.push(post);
if (posts.length >= TIMELINE_CAP) break;
}
const articles = posts.filter((p) => p.article).map((p) => describe(p, me.username));
console.log(`@${me.username}: ${posts.length} posts read (oldest ${posts.at(-1)?.created_at ?? 'none'}), ${articles.length} Articles`);
// The snapshot is rewritten after every Article, so a run cut short by a
// long rate-limit wait still leaves everything it already collected.
await mkdir(OUT_DIR, { recursive: true });
const file = path.join(OUT_DIR, `${me.username}-articles-${new Date().toISOString().slice(0, 10)}.json`);
const save = (complete) =>
writeFile(file, `${JSON.stringify({ source: 'x-api-v2', handle: me.username, fetchedAt: new Date().toISOString(), complete, timelinePostsRead: posts.length, articles }, null, '\t')}\n`);
await save(!WITH_REPLIES);
if (WITH_REPLIES) {
for (const article of articles) {
Object.assign(article, await readerResponse(client, article, me.id));
await save(false);
console.log(` ${article.created_at.slice(0, 10)} ${String(article.replies.length).padStart(3)} replies ${String(article.quotes.length).padStart(3)} quotes ${article.title}`);
}
await save(true);
}
console.log(`wrote ${path.relative(REPO_ROOT, file)}`);
}
main().catch((err) => {
console.error(`x:articles:pull: ${err?.data?.title || err?.message || err}`);
process.exit(1);
});