Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(x-archive): pull every published X Article with its full text and reader replies
nirholas4ad3e16942be
diff --git a/data/x-archive/README.md b/data/x-archive/README.mdindex 889348581..a5029ff10 100644--- a/data/x-archive/README.md+++ b/data/x-archive/README.md@@ -10,6 +10,7 @@ its API terms again.|---|---|| `<handle>-<YYYY-MM-DD>.json` | A timeline scrape: the whole profile as rendered on the web, including reposts of other accounts. || `<handle>-api-<YYYY-MM-DD>.json` | A metrics refresh written by `npm run x:archive:refresh`: the same shape, with exact counters from the X API v2. Carries `"source": "x-api-v2"`. |+| `<handle>-articles-<YYYY-MM-DD>.json` | Every X Article the account has published, written by `npm run x:articles:pull`: title, preview, full plain text, exact metrics, and the replies and quote posts readers left under each one. Carries `"source": "x-api-v2"`. || `analysis/<handle>-engagement.json` | The computed analysis behind the readable report in [docs/x-archive/](../../docs/x-archive/). Generated, not hand-edited. (Named `analysis/`, not `reports/`, because the repo's `.gitignore` drops any directory called `reports/`.) |Snapshots are additive. Never overwrite an old one to "update" a post: the@@ -25,6 +26,26 @@ npm run x:archive:refresh # pull exact metrics from the X API into a new snapnpm run x:archive:analyze # rebuild docs/x-archive/<handle>-engagement.md```+A fourth command pulls the long-form side of the account:++```bash+npm run x:articles:pull # every Article, its full text, its replies and quotes+npm run x:articles:pull -- --no-replies # Articles and metrics only (no search calls)+npm run x:articles:pull -- --since 2026-08-01+```++A scrape renders an Article as a title card. The API returns the whole body when+a post is read with the `article` field, and full-archive search returns the+reply thread, so the snapshot holds both what the account wrote and what readers+asked for in response. It reads the posting account's OAuth credentials from the+environment or, failing that, from the Cloud Run service. Search is rate limited+to a small budget per 15 minutes, and the script waits out each limit instead of+failing, so a full run can take several minutes.++Reader replies are other people's words and often name other tokens. Under the+commit gate in CLAUDE.md, a snapshot that references another crypto project+needs owner approval before it is committed.+`import` hashes each file and records it in `x_account_imports`, so running ittwice is a no-op. `analyze` reads the database when `DATABASE_URL` is set andfalls back to these files when it is not, which is what makes the archive thediff --git a/package.json b/package.jsonindex 9f3f6ae7d..058e9a254 100644--- a/package.json+++ b/package.json@@ -219,6 +219,7 @@"x:archive:import": "node scripts/x-archive-import.mjs","x:archive:refresh": "node scripts/x-archive-refresh.mjs","x:archive:analyze": "node scripts/x-archive-analyze.mjs",+ "x:articles:pull": "node scripts/x-articles-pull.mjs","x:volume": "node scripts/x-volume-study.mjs","audit:deploy": "node scripts/audit-deploy-artifacts.mjs","guard:esbuild": "node scripts/guard-esbuild-bundles.mjs",diff --git a/scripts/x-articles-pull.mjs b/scripts/x-articles-pull.mjsnew file mode 100644index 000000000..cdbb298da--- /dev/null+++ b/scripts/x-articles-pull.mjs@@ -0,0 +1,168 @@+#!/usr/bin/env node+/**+ * Pull every X Article an account has published, with what readers said back.+ *+ * A timeline scrape sees an Article as a title card. The X API v2 returns the+ * whole thing when a post is read with the `article` tweet field: title,+ * preview, cover, and the full plain text with its code blocks and links. This+ * script walks the account's timeline (the API returns the last 3,200 posts),+ * keeps the posts that carry an Article, and for each one collects the replies+ * in its conversation (full-archive search) and the posts that quote it. Those+ * two lists are the closest thing the account has to a reader survey: what+ * people asked, what they pushed back on, what they wanted next.+ *+ * Usage:+ * npm run x:articles:pull # every Article on @trythreews+ * npm run x:articles:pull -- --no-replies # Articles and metrics only+ * npm run x:articles:pull -- --since 2026-08-01+ *+ * Output: data/x-archive/<handle>-articles-<YYYY-MM-DD>.json. Snapshots are+ * additive, like the rest of the archive (see data/x-archive/README.md).+ *+ * Credentials: X_API_KEY, X_API_SECRET, X_ACCESS_TOKEN, X_ACCESS_SECRET, the+ * posting account's OAuth 1.0a user context. Read from the environment, then+ * .env.local and .env, then the Cloud Run service through+ * scripts/read-service-env.mjs. Read-only: nothing is posted.+ */++import { execFileSync } from 'node:child_process';+import { readFileSync } from 'node:fs';+import { mkdir, writeFile } from 'node:fs/promises';+import path from 'node:path';+import { fileURLToPath } from 'node:url';+import { xClientFromEnv } from '../api/_lib/x-content/publisher.js';++const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');+const OUT_DIR = path.join(REPO_ROOT, 'data', 'x-archive');+const CREDENTIALS = ['X_API_KEY', 'X_API_SECRET', 'X_ACCESS_TOKEN', 'X_ACCESS_SECRET'];+const TIMELINE_CAP = 3200;++const args = process.argv.slice(2);+const flag = (name) => args.includes(`--${name}`);+const value = (name) => {+ const i = args.indexOf(`--${name}`);+ return i >= 0 ? args[i + 1] : null;+};+const SINCE = value('since');+const WITH_REPLIES = !flag('no-replies');++function loadEnvFiles() {+ for (const file of ['.env.local', '.env']) {+ let raw;+ try {+ raw = readFileSync(path.join(REPO_ROOT, file), 'utf8');+ } catch {+ continue;+ }+ for (const line of raw.split('\n')) {+ const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);+ if (!m || process.env[m[1]]) continue;+ process.env[m[1]] = m[2].trim().replace(/^(['"])(.*)\1$/, '$2');+ }+ }+}++function loadServiceCredentials() {+ for (const name of CREDENTIALS) {+ if (process.env[name]) continue;+ const out = execFileSync('node', ['scripts/read-service-env.mjs', `^${name}$`, '--raw'], { cwd: REPO_ROOT, encoding: 'utf8' }).trim();+ if (out) process.env[name] = out;+ }+}++const TWEET_FIELDS = ['public_metrics', 'created_at', 'article', 'note_tweet', 'entities', 'referenced_tweets', 'conversation_id', 'author_id', 'in_reply_to_user_id'];+const READER_QUERY = {+ 'tweet.fields': ['public_metrics', 'created_at', 'note_tweet', 'referenced_tweets', 'author_id', 'in_reply_to_user_id'],+ expansions: ['author_id'],+ 'user.fields': ['username', 'name', 'public_metrics', 'verified'],+ max_results: 100,+};++async function collect(paginator) {+ const posts = [];+ for await (const post of paginator) posts.push(post);+ const users = new Map((paginator.includes?.users || []).map((u) => [u.id, u]));+ return posts.map((post) => {+ const author = users.get(post.author_id);+ return {+ id: post.id,+ url: author ? `https://x.com/${author.username}/status/${post.id}` : `https://x.com/i/status/${post.id}`,+ created_at: post.created_at,+ author: { id: post.author_id, username: author?.username ?? null, name: author?.name ?? null, followers: author?.public_metrics?.followers_count ?? null },+ text: post.note_tweet?.text || post.text,+ metrics: post.public_metrics,+ };+ });+}++// Full-archive search allows one request a second and a small budget per 15+// minutes. On a 429 wait until the reset X reports, then run the step again.+async function withRateLimit(label, step) {+ for (;;) {+ try {+ return await step();+ } catch (err) {+ if (err?.code !== 429) throw err;+ const reset = Number(err.rateLimit?.reset || 0) * 1000;+ const waitMs = Math.max(15_000, reset - Date.now() + 2_000);+ console.warn(` rate limited on ${label}; waiting ${Math.round(waitMs / 1000)}s`);+ await new Promise((resolve) => setTimeout(resolve, waitMs));+ }+ }+}++async function readerResponse(client, article, selfId) {+ const replies = await withRateLimit('replies', async () => collect(await client.searchAll(`conversation_id:${article.id} -from:${selfId}`, { ...READER_QUERY, start_time: article.created_at })));+ const quotes = await withRateLimit('quotes', async () => collect(await client.quotes(article.id, READER_QUERY)));+ return { replies, quotes: quotes.filter((q) => q.author.id !== selfId) };+}++function describe(post, handle) {+ const a = post.article;+ return {+ id: post.id,+ url: `https://x.com/${handle}/status/${post.id}`,+ created_at: post.created_at,+ title: a.title?.trim() || null,+ preview: a.preview_text || null,+ cover_media: a.cover_media || null,+ characters: (a.plain_text || '').length,+ metrics: post.public_metrics,+ body: a.plain_text || null,+ entities: a.entities || null,+ };+}++async function main() {+ loadEnvFiles();+ loadServiceCredentials();+ const client = await xClientFromEnv();+ if (!client) throw new Error(`missing X credentials: set ${CREDENTIALS.join(', ')} or authenticate gcloud so the Cloud Run copies can be read`);++ const { data: me } = await client.me();+ const timeline = await client.userTimeline(me.id, { 'tweet.fields': TWEET_FIELDS, max_results: 100, exclude: ['retweets'], ...(SINCE ? { start_time: new Date(SINCE).toISOString() } : {}) });+ const posts = [];+ for await (const post of timeline) {+ posts.push(post);+ if (posts.length >= TIMELINE_CAP) break;+ }+ const articles = posts.filter((p) => p.article).map((p) => describe(p, me.username));+ console.log(`@${me.username}: ${posts.length} posts read (oldest ${posts.at(-1)?.created_at ?? 'none'}), ${articles.length} Articles`);++ if (WITH_REPLIES) {+ for (const article of articles) {+ Object.assign(article, await readerResponse(client, article, me.id));+ console.log(` ${article.created_at.slice(0, 10)} ${String(article.replies.length).padStart(3)} replies ${String(article.quotes.length).padStart(3)} quotes ${article.title}`);+ }+ }++ await mkdir(OUT_DIR, { recursive: true });+ const file = path.join(OUT_DIR, `${me.username}-articles-${new Date().toISOString().slice(0, 10)}.json`);+ await writeFile(file, `${JSON.stringify({ source: 'x-api-v2', handle: me.username, fetchedAt: new Date().toISOString(), timelinePostsRead: posts.length, articles }, null, '\t')}\n`);+ console.log(`wrote ${path.relative(REPO_ROOT, file)}`);+}++main().catch((err) => {+ console.error(`x:articles:pull: ${err?.data?.title || err?.message || err}`);+ process.exit(1);+});