Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(x-content): read only the posts whose numbers can still move when the queue learns

nirholasa2ebec2457c4
diff --git a/api/_lib/x-content/outcomes.js b/api/_lib/x-content/outcomes.js
index 92cd1d602..2947c3fc7 100644
--- a/api/_lib/x-content/outcomes.js
+++ b/api/_lib/x-content/outcomes.js
@@ -33,6 +33,12 @@ export const OUTCOMES_WINDOW_DAYS = 90;
export const OUTCOMES_LIMIT = 800;
// A stored record older than this is read again on the next publishing tick.
export const OUTCOMES_REFRESH_HOURS = 6;
+// X bills every post a read returns, so a refresh only reads back as far as
+// metrics can still be moving. A post's likes and bookmarks settle within a few
+// days, so anything older than this is kept from the stored record instead of
+// being read, and paid for, again. The first read, or one after a long gap,
+// still reads the whole window.
+export const OUTCOMES_SETTLE_DAYS = 7;
// X answers 503 on a timeline page often enough that one failed page should not
// cost the whole read. The wait grows with each try.
export const READ_ATTEMPTS = 3;
@@ -273,6 +279,15 @@ const isTransient = (err) => Number(err?.code ?? err?.status) >= 500;
const wait = (ms) => new Promise((done) => setTimeout(done, ms));
+// How far back this refresh has to read: the whole window the first time, and
+// otherwise only from just before the last read, less the days a post's
+// metrics can still move.
+export function readDaysFor(previous, { now = Date.now(), days = OUTCOMES_WINDOW_DAYS, settleDays = OUTCOMES_SETTLE_DAYS } = {}) {
+ const fetched = Date.parse(previous?.fetchedAt || '');
+ if (!previous?.posts?.length || !Number.isFinite(fetched) || fetched > now) return days;
+ return Math.min(days, (now - fetched) / DAY + settleDays);
+}
+
export async function collectOutcomes({
client,
ledger = null,
@@ -281,13 +296,16 @@ export async function collectOutcomes({
limit = OUTCOMES_LIMIT,
attempts = READ_ATTEMPTS,
retryDelayMs = READ_RETRY_MS,
+ previous = null,
+ settleDays = OUTCOMES_SETTLE_DAYS,
}) {
+ const readDays = readDaysFor(previous, { now, days, settleDays });
// A read that fails part way is started again from the top: half a timeline
// would measure the recent posts against nothing.
let read = null;
for (let attempt = 1; !read; attempt++) {
try {
- read = await readTimeline({ client, now, days, limit });
+ read = await readTimeline({ client, now, days: readDays, limit });
} catch (err) {
if (attempt >= attempts || !isTransient(err)) throw err;
await wait(retryDelayMs * attempt);
@@ -316,15 +334,28 @@ export async function collectOutcomes({
for (const id of [leadPostId(row), ...(row.postIds || [])]) if (id) pipelineById.set(String(id), row);
}
+ const fresh = heads.map((tweet) => describePost(tweet, { mediaByKey, threadSizes, pipelineById }));
+ // Settled rows come from the stored record. A stored row inside the window
+ // just read that the read did not return was deleted, so it is dropped.
+ const readFrom = now - readDays * DAY;
+ const windowFrom = now - days * DAY;
+ const kept = (previous?.posts || []).filter((row) => {
+ const at = Date.parse(row.at);
+ return at >= windowFrom && at < readFrom;
+ });
return {
fetchedAt: new Date(now).toISOString(),
- posts: heads.map((tweet) => describePost(tweet, { mediaByKey, threadSizes, pipelineById })),
+ readDays: Math.round(readDays * 10) / 10,
+ posts: [...fresh, ...kept].sort((a, b) => Date.parse(b.at) - Date.parse(a.at)),
};
}
+// A failed read counts as a read for pacing: retrying every fifteen minutes
+// would only repeat the refusal (a 402 stays a 402 until the account is topped
+// up) and log it each time.
export function outcomesAreStale(outcomes, now = Date.now(), hours = OUTCOMES_REFRESH_HOURS) {
- const fetched = Date.parse(outcomes?.fetchedAt || '');
- return !Number.isFinite(fetched) || now - fetched > hours * HOUR;
+ const last = Math.max(Date.parse(outcomes?.fetchedAt || '') || 0, Date.parse(outcomes?.attemptedAt || '') || 0);
+ return !last || now - last > hours * HOUR;
}
// ── Learning ────────────────────────────────────────────────────────────────
diff --git a/api/_lib/x-content/runner.js b/api/_lib/x-content/runner.js
index f454dcbfa..45dbb247c 100644
--- a/api/_lib/x-content/runner.js
+++ b/api/_lib/x-content/runner.js
@@ -122,11 +122,12 @@ async function refreshOutcomes({ outcomes, stored, collect, client, ledger, now
const sample = (rows) => learnLifts(rows, { now }).sample;
if (!outcomes || !outcomesAreStale(stored, now)) return { refreshed: false, sample: sample(stored?.posts) };
try {
- const fresh = await collect({ client, ledger, now });
+ const fresh = await collect({ client, ledger, now, previous: stored });
await outcomes.save(fresh);
- return { refreshed: true, sample: sample(fresh.posts) };
+ return { refreshed: true, sample: sample(fresh.posts), readDays: fresh.readDays };
} catch (err) {
console.warn('[x-content] outcomes refresh failed', err?.message || err);
+ await outcomes.save({ ...(stored || { fetchedAt: null, posts: [] }), attemptedAt: new Date(now).toISOString(), attemptError: String(err?.message || err).slice(0, 200) }).catch(() => {});
return { refreshed: false, sample: sample(stored?.posts) };
}
}
diff --git a/tests/x-content-outcomes.test.js b/tests/x-content-outcomes.test.js
index 5ce446382..f317f17cc 100644
--- a/tests/x-content-outcomes.test.js
+++ b/tests/x-content-outcomes.test.js
@@ -20,6 +20,9 @@ import {
memoryOutcomesStore,
outcomeOf,
outcomesAreStale,
+ readDaysFor,
+ OUTCOMES_SETTLE_DAYS,
+ OUTCOMES_WINDOW_DAYS,
} from '../api/_lib/x-content/outcomes.js';
import { loadLifts, scoreItem } from '../api/_lib/x-content/priority.js';
import { runTick } from '../api/_lib/x-content/runner.js';
@@ -276,6 +279,57 @@ describe('collectOutcomes', () => {
});
});
+describe('incremental refresh', () => {
+ const stored = (fetchedDaysAgo, posts) => ({ fetchedAt: daysAgo(fetchedDaysAgo), posts });
+
+ it('reads the whole window the first time and after a long gap', () => {
+ expect(readDaysFor(null, { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);
+ expect(readDaysFor({ fetchedAt: null, posts: [] }, { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);
+ expect(readDaysFor(stored(1, []), { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);
+ expect(readDaysFor(stored(200, [row()]), { now: NOW })).toBe(OUTCOMES_WINDOW_DAYS);
+ });
+
+ it('otherwise reads back only as far as metrics can still move', () => {
+ expect(readDaysFor(stored(0.25, [row()]), { now: NOW })).toBeCloseTo(0.25 + OUTCOMES_SETTLE_DAYS);
+ expect(readDaysFor(stored(2, [row()]), { now: NOW })).toBeCloseTo(2 + OUTCOMES_SETTLE_DAYS);
+ });
+
+ it('keeps settled rows, takes moving rows from the read, and drops what was deleted or left the window', async () => {
+ const settled = row({ id: 's1', at: daysAgo(30), likes: 90 });
+ const outOfWindow = row({ id: 'old', at: daysAgo(OUTCOMES_WINDOW_DAYS + 5) });
+ const deleted = row({ id: 'gone', at: daysAgo(2) });
+ const remeasured = row({ id: '300', at: daysAgo(3), likes: 1 });
+ const calls = [];
+ const client = {
+ async me() {
+ return { data: { id: '42' } };
+ },
+ async userTimeline(id, options) {
+ calls.push(options.start_time);
+ return {
+ includes: { media: [] },
+ async *[Symbol.asyncIterator]() {
+ yield tweet('300', 'A post whose likes are still arriving', { created_at: daysAgo(3), public_metrics: metrics(40) });
+ },
+ };
+ },
+ };
+ const previous = stored(0.25, [remeasured, deleted, settled, outOfWindow]);
+ const outcomes = await collectOutcomes({ client, now: NOW, previous });
+
+ expect(outcomes.readDays).toBeCloseTo(0.25 + OUTCOMES_SETTLE_DAYS, 1);
+ expect(Date.parse(calls[0])).toBeGreaterThan(NOW - (OUTCOMES_SETTLE_DAYS + 1) * DAY);
+ expect(outcomes.posts.map((post) => post.id)).toEqual(['300', 's1']);
+ expect(outcomes.posts[0].likes).toBe(40);
+ });
+
+ it('paces a failed read like a read, so a refusal is not repeated every tick', () => {
+ const failed = { fetchedAt: daysAgo(2), attemptedAt: new Date(NOW - HOUR).toISOString(), posts: [] };
+ expect(outcomesAreStale(failed, NOW)).toBe(false);
+ expect(outcomesAreStale(failed, NOW + OUTCOMES_REFRESH_HOURS * HOUR)).toBe(true);
+ });
+});
+
describe('learnLifts', () => {
it('scores a post on likes, bookmarks, and reposts, and ignores replies', () => {
const base = row({ likes: 20, bookmarks: 6, reposts: 3 });
@@ -543,7 +597,7 @@ describe('runTick and the outcomes loop', () => {
expect(result.published.slot).toBe('2026-09-17#1');
expect(events).toEqual(['tweet', 'collect after rigged']);
expect(collect).toHaveBeenCalledTimes(1);
- expect(collect.mock.calls[0][0]).toMatchObject({ client, now: NOW });
+ expect(collect.mock.calls[0][0]).toMatchObject({ client, now: NOW, previous: await stale().load() });
expect(result.outcomes).toEqual({ refreshed: true, sample: 25 });
expect(await outcomes.load()).toEqual(fresh);
});
@@ -576,7 +630,11 @@ describe('runTick and the outcomes loop', () => {
expect(result.outcomes).toEqual({ refreshed: false, sample: 24 });
expect((await store.load()).published.map((entry) => entry.id)).toEqual(['rigged']);
expect(warn).toHaveBeenCalledWith('[x-content] outcomes refresh failed', 'Too Many Requests');
- expect((await outcomes.load()).posts).toHaveLength(24);
+ const after = await outcomes.load();
+ expect(after.posts).toHaveLength(24);
+ expect(after.attemptedAt).toBe(new Date(NOW).toISOString());
+ expect(after.attemptError).toBe('Too Many Requests');
+ expect(outcomesAreStale(after, NOW + HOUR)).toBe(false);
} finally {
warn.mockRestore();
}