Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(x-content): read only the posts whose numbers can still move when the queue learns
nirholasa2ebec2457c4
diff --git a/api/_lib/x-content/outcomes.js b/api/_lib/x-content/outcomes.jsindex 92cd1d602..2947c3fc7 100644--- a/api/_lib/x-content/outcomes.js+++ b/api/_lib/x-content/outcomes.js@@ -33,6 +33,12 @@ export const OUTCOMES_WINDOW_DAYS = 90;export const OUTCOMES_LIMIT = 800;// A stored record older than this is read again on the next publishing tick.export const OUTCOMES_REFRESH_HOURS = 6;+// X bills every post a read returns, so a refresh only reads back as far as+// metrics can still be moving. A post's likes and bookmarks settle within a few+// days, so anything older than this is kept from the stored record instead of+// being read, and paid for, again. The first read, or one after a long gap,+// still reads the whole window.+export const OUTCOMES_SETTLE_DAYS = 7;// X answers 503 on a timeline page often enough that one failed page should not// cost the whole read. The wait grows with each try.export const READ_ATTEMPTS = 3;@@ -273,6 +279,15 @@ const isTransient = (err) => Number(err?.code ?? err?.status) >= 500;const wait = (ms) => new Promise((done) => setTimeout(done, ms));+// How far back this refresh has to read: the whole window the first time, and+// otherwise only from just before the last read, less the days a post's+// metrics can still move.+export function readDaysFor(previous, { now = Date.now(), days = OUTCOMES_WINDOW_DAYS, settleDays = OUTCOMES_SETTLE_DAYS } = {}) {+ const fetched = Date.parse(previous?.fetchedAt || '');+ if (!previous?.posts?.length || !Number.isFinite(fetched) || fetched > now) return days;+ return Math.min(days, (now - fetched) / DAY + settleDays);+}+export async function collectOutcomes({client,ledger = null,@@ -281,13 +296,16 @@ export async function collectOutcomes({limit = OUTCOMES_LIMIT,attempts = READ_ATTEMPTS,retryDelayMs = READ_RETRY_MS,+ previous = null,+ settleDays = OUTCOMES_SETTLE_DAYS,}) {+ const readDays = readDaysFor(previous, { now, days, settleDays });// A read that fails part way is started again from the top: half a timeline// would measure the recent posts against nothing.let read = null;for (let attempt = 1; !read; attempt++) {try {- read = await readTimeline({ client, now, days, limit });+ read = await readTimeline({ client, now, days: readDays, limit });} catch (err) {if (attempt >= attempts || !isTransient(err)) throw err;await wait(retryDelayMs * attempt);@@ -316,15 +334,28 @@ export async function collectOutcomes({for (const id of [leadPostId(row), ...(row.postIds || [])]) if (id) pipelineById.set(String(id), row);}+ const fresh = heads.map((tweet) => describePost(tweet, { mediaByKey, threadSizes, pipelineById }));+ // Settled rows come from the stored record. A stored row inside the window+ // just read that the read did not return was deleted, so it is dropped.+ const readFrom = now - readDays * DAY;+ const windowFrom = now - days * DAY;+ const kept = (previous?.posts || []).filter((row) => {+ const at = Date.parse(row.at);+ return at >= windowFrom && at < readFrom;+ });return {fetchedAt: new Date(now).toISOString(),- posts: heads.map((tweet) => describePost(tweet, { mediaByKey, threadSizes, pipelineById })),+ readDays: Math.round(readDays * 10) / 10,+ posts: [...fresh, ...kept].sort((a, b) => Date.parse(b.at) - Date.parse(a.at)),};}+// A failed read counts as a read for pacing: retrying every fifteen minutes+// would only repeat the refusal (a 402 stays a 402 until the account is topped+// up) and log it each time.export function outcomesAreStale(outcomes, now = Date.now(), hours = OUTCOMES_REFRESH_HOURS) {- const fetched = Date.parse(outcomes?.fetchedAt || '');- return !Number.isFinite(fetched) || now - fetched > hours * HOUR;+ const last = Math.max(Date.parse(outcomes?.fetchedAt || '') || 0, Date.parse(outcomes?.attemptedAt || '') || 0);+ return !last || now - last > hours * HOUR;}// ── Learning ────────────────────────────────────────────────────────────────diff --git a/api/_lib/x-content/runner.js b/api/_lib/x-content/runner.jsindex f454dcbfa..45dbb247c 100644--- a/api/_lib/x-content/runner.js+++ b/api/_lib/x-content/runner.js@@ -122,11 +122,12 @@ async function refreshOutcomes({ outcomes, stored, collect, client, ledger, nowconst sample = (rows) => learnLifts(rows, { now }).sample;if (!outcomes || !outcomesAreStale(stored, now)) return { refreshed: false, sample: sample(stored?.posts) };try {- const fresh = await collect({ client, ledger, now });+ const fresh = await collect({ client, ledger, now, previous: stored });await outcomes.save(fresh);- return { refreshed: true, sample: sample(fresh.posts) };+ return { refreshed: true, sample: sample(fresh.posts), readDays: fresh.readDays };} catch (err) {console.warn('[x-content] outcomes refresh failed', err?.message || err);+ await outcomes.save({ ...(stored || { fetchedAt: null, posts: [] }), attemptedAt: new Date(now).toISOString(), attemptError: String(err?.message || err).slice(0, 200) }).catch(() => {});return { refreshed: false, sample: sample(stored?.posts) };}}diff --git a/tests/x-content-outcomes.test.js b/tests/x-content-outcomes.test.jsindex 5ce446382..f317f17cc 100644--- a/tests/x-content-outcomes.test.js+++ b/tests/x-content-outcomes.test.js@@ -20,6 +20,9 @@ import {memoryOutcomesStore,outcomeOf,outcomesAreStale,+ readDaysFor,+ OUTCOMES_SETTLE_DAYS,+ OUTCOMES_WINDOW_DAYS,} from '../api/_lib/x-content/outcomes.js';import { loadLifts, scoreItem } from '../api/_lib/x-content/priority.js';import { runTick } from '../api/_lib/x-content/runner.js';@@ -276,6 +279,57 @@ describe('collectOutcomes', () => {});});+describe('incremental refresh', () => {+ const stored = (fetchedDaysAgo, posts) => ({ fetchedAt: daysAgo(fetchedDaysAgo), posts });++ it('reads the whole window the first time and after a long gap', () => {+ expect(readDaysFor(null, { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);+ expect(readDaysFor({ fetchedAt: null, posts: [] }, { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);+ expect(readDaysFor(stored(1, []), { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);+ expect(readDaysFor(stored(200, [row()]), { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);+ });++ it('otherwise reads back only as far as metrics can still move', () => {+ expect(readDaysFor(stored(0.25, [row()]), { now: NOW })).toBeCloseTo(0.25 + OUTCOMES_SETTLE_DAYS);+ expect(readDaysFor(stored(2, [row()]), { now: NOW })).toBeCloseTo(2 + OUTCOMES_SETTLE_DAYS);+ });++ it('keeps settled rows, takes moving rows from the read, and drops what was deleted or left the window', async () => {+ const settled = row({ id: 's1', at: daysAgo(30), likes: 90 });+ const outOfWindow = row({ id: 'old', at: daysAgo(OUTCOMES_WINDOW_DAYS + 5) });+ const deleted = row({ id: 'gone', at: daysAgo(2) });+ const remeasured = row({ id: '300', at: daysAgo(3), likes: 1 });+ const calls = [];+ const client = {+ async me() {+ return { data: { id: '42' } };+ },+ async userTimeline(id, options) {+ calls.push(options.start_time);+ return {+ includes: { media: [] },+ async *[Symbol.asyncIterator]() {+ yield tweet('300', 'A post whose likes are still arriving', { created_at: daysAgo(3), public_metrics: metrics(40) });+ },+ };+ },+ };+ const previous = stored(0.25, [remeasured, deleted, settled, outOfWindow]);+ const outcomes = await collectOutcomes({ client, now: NOW, previous });++ expect(outcomes.readDays).toBeCloseTo(0.25 + OUTCOMES_SETTLE_DAYS, 1);+ expect(Date.parse(calls[0])).toBeGreaterThan(NOW - (OUTCOMES_SETTLE_DAYS + 1) * DAY);+ expect(outcomes.posts.map((post) => post.id)).toEqual(['300', 's1']);+ expect(outcomes.posts[0].likes).toBe(40);+ });++ it('paces a failed read like a read, so a refusal is not repeated every tick', () => {+ const failed = { fetchedAt: daysAgo(2), attemptedAt: new Date(NOW - HOUR).toISOString(), posts: [] };+ expect(outcomesAreStale(failed, NOW)).toBe(false);+ expect(outcomesAreStale(failed, NOW + OUTCOMES_REFRESH_HOURS * HOUR)).toBe(true);+ });+});+describe('learnLifts', () => {it('scores a post on likes, bookmarks, and reposts, and ignores replies', () => {const base = row({ likes: 20, bookmarks: 6, reposts: 3 });@@ -543,7 +597,7 @@ describe('runTick and the outcomes loop', () => {expect(result.published.slot).toBe('2026-09-17#1');expect(events).toEqual(['tweet', 'collect after rigged']);expect(collect).toHaveBeenCalledTimes(1);- expect(collect.mock.calls[0][0]).toMatchObject({ client, now: NOW });+ expect(collect.mock.calls[0][0]).toMatchObject({ client, now: NOW, previous: await stale().load() });expect(result.outcomes).toEqual({ refreshed: true, sample: 25 });expect(await outcomes.load()).toEqual(fresh);});@@ -576,7 +630,11 @@ describe('runTick and the outcomes loop', () => {expect(result.outcomes).toEqual({ refreshed: false, sample: 24 });expect((await store.load()).published.map((entry) => entry.id)).toEqual(['rigged']);expect(warn).toHaveBeenCalledWith('[x-content] outcomes refresh failed', 'Too Many Requests');- expect((await outcomes.load()).posts).toHaveLength(24);+ const after = await outcomes.load();+ expect(after.posts).toHaveLength(24);+ expect(after.attemptedAt).toBe(new Date(NOW).toISOString());+ expect(after.attemptError).toBe('Too Many Requests');+ expect(outcomesAreStale(after, NOW + HOUR)).toBe(false);} finally {warn.mockRestore();}