Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(ship-feed): serve last-good on GitHub rate limit instead of throwing

nirholasbaafb0a4c7eb
diff --git a/api/_lib/commit-feed-push.js b/api/_lib/commit-feed-push.js
index a71f3551f..bdb037398 100644
--- a/api/_lib/commit-feed-push.js
+++ b/api/_lib/commit-feed-push.js
@@ -128,6 +128,23 @@ export async function releaseLock() {
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+// Tags a GitHub rate-limit exhaustion so callers can match on a stable flag
+// instead of pattern-matching the message string. A shared Cloud Run egress
+// IP can burn through the unauthenticated 60/hour ceiling without this cron
+// doing anything wrong, so this is an expected, no-fault condition, not a
+// bug: callers are expected to skip gracefully, not propagate it as a 5xx.
+export function githubRateLimitError(resetMs) {
+ const until = Number.isFinite(resetMs) ? new Date(resetMs).toISOString() : 'unknown';
+ return Object.assign(
+ new Error(`GitHub rate limit exhausted (resets ${until}); set GITHUB_TOKEN to raise it above 60/hour`),
+ { githubRateLimited: true, retryAfter: Number.isFinite(resetMs) ? until : null },
+ );
+}
+
+export function isGithubRateLimitError(err) {
+ return Boolean(err?.githubRateLimited);
+}
+
async function fetchCommitPage(page) {
const headers = { accept: 'application/vnd.github+json', 'user-agent': 'three.ws-commit-feed' };
const token = process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '';
@@ -141,10 +158,7 @@ async function fetchCommitPage(page) {
// without this cron doing anything wrong.
if (res.headers.get('x-ratelimit-remaining') === '0') {
const reset = Number(res.headers.get('x-ratelimit-reset')) * 1000;
- const until = Number.isFinite(reset) ? new Date(reset).toISOString() : 'unknown';
- throw new Error(
- `GitHub rate limit exhausted (resets ${until}); set GITHUB_TOKEN to raise it above 60/hour`,
- );
+ throw githubRateLimitError(reset);
}
throw new Error(`GitHub commits fetch failed (${res.status})`);
}
@@ -272,13 +286,22 @@ async function sendTelegram(botToken, chatId, text, previewUrl) {
}
}
-export async function pushTelegramLane() {
+export async function pushTelegramLane({ getPage = fetchCommitPage } = {}) {
const botToken = process.env.TELEGRAM_BOT_TOKEN;
const chatId = process.env.TELEGRAM_COMMITS_CHAT_ID || process.env.TELEGRAM_CHANGELOG_CHAT_ID;
if (!botToken || !chatId) return { skipped: 'not_configured' };
const state = (await getState(STATE_KEY)) || { lastSha: null, lastDate: null };
- const { commits, found } = await fetchCommitsSince(state.lastSha);
+ let commits, found;
+ try {
+ ({ commits, found } = await fetchCommitsSince(state.lastSha, { getPage }));
+ } catch (err) {
+ if (isGithubRateLimitError(err)) {
+ console.warn(`[commit-feed] ${err.message}`);
+ return { posted: 0, skipped: 'github_rate_limited', retryAfter: err.retryAfter };
+ }
+ throw err;
+ }
if (commits.length === 0) return { posted: 0 };
if (state.lastSha && !found) {
diff --git a/api/ship/feed.js b/api/ship/feed.js
index 9bf95fa29..27da0d971 100644
--- a/api/ship/feed.js
+++ b/api/ship/feed.js
@@ -36,7 +36,7 @@ import { join } from 'node:path';
import { cors, json, method, wrap, rateLimited, error } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
-import { cacheGet, cacheSet } from '../_lib/cache.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
import { fetchUpstream } from '../_lib/upstream-fetch.js';
import {
buildShipFeed,
@@ -106,29 +106,32 @@ const boundedFetch = (url, init) =>
async function buildFeed({ limit, audience }) {
const key = `ship:feed:${limit}:${audience}`;
- const cached = await cacheGet(key);
- if (cached) return { ...cached, cached: true };
-
- const entries = loadChangelog();
- const commits = await fetchGitHubCommits({
- repo: REPO,
- branch: 'main',
- limit,
- token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '',
- fetchImpl: boundedFetch,
- userAgent: 'three.ws-shipfeed',
- });
-
- const feed = buildShipFeed({
- commits,
- entries,
- repo: REPO,
- siteUrl: SITE,
- minAudience: audience,
- classify: { productScopes: PRODUCT_SCOPES },
- });
- await cacheSet(key, feed, CACHE_TTL_S);
- return { ...feed, cached: false };
+ const { value, stale } = await cacheWrapLastGood(
+ key,
+ CACHE_TTL_S,
+ async () => {
+ const entries = loadChangelog();
+ const commits = await fetchGitHubCommits({
+ repo: REPO,
+ branch: 'main',
+ limit,
+ token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '',
+ fetchImpl: boundedFetch,
+ userAgent: 'three.ws-shipfeed',
+ });
+
+ return buildShipFeed({
+ commits,
+ entries,
+ repo: REPO,
+ siteUrl: SITE,
+ minAudience: audience,
+ classify: { productScopes: PRODUCT_SCOPES },
+ });
+ },
+ { withMeta: true },
+ );
+ return { ...value, cached: false, stale };
}
/**
diff --git a/prompts/finish/_context/production-100-OWNER-ACTIONS.md b/prompts/finish/_context/production-100-OWNER-ACTIONS.md
index 34942399a..a79c67366 100644
--- a/prompts/finish/_context/production-100-OWNER-ACTIONS.md
+++ b/prompts/finish/_context/production-100-OWNER-ACTIONS.md
@@ -86,6 +86,7 @@ on it; some resolve themselves as other work lands.
| 67 | Deploy `workers/model-triposplat` (`gcloud builds submit --config workers/model-triposplat/cloudbuild.yaml`, stage weights with `stage-assets.sh`), set `MODEL_TRIPOSPLAT_URL` on `three-ws-api` with `--update-env-vars`, then run `POST /api/forge-splat` on a real photo and open the returned splat in `/splat`. Added 2026-10-23, order 081 close-out. | minutes | Image-to-splat Forge lane live (order 081 DoD line 2) | `workers/model-triposplat/README.md` |
| 68 | world.three.ws has been unreachable since the backing Cloud Run service `hyperfy-world` was deleted via the Cloud Run API by `[email protected]` at 2026-10-23T02:48:38Z (confirmed via `gcloud logging read 'protoPayload.methodName="google.cloud.run.v1.Services.DeleteService"'`), a deliberate recent action, not drift. The `world.three.ws` domain mapping is itself still healthy and provisioned; it now points at a service that no longer exists, which is why it answers a generic 404 instead of a branded error. Decide: redeploy (`bash deploy/world/apply-hardening.sh`, fully scripted and idempotent, documented in `deploy/world/README.md`) or formally retire the surface (remove the domain mapping and any in-product links to it). `/api/healthz` now correctly reports this as `world: down` instead of masking it as `unknown` (order 413). | decision | world.three.ws restoration or retirement | `deploy/world/README.md` |
| 69 | **Redeploy `remesh-service`.** Added 2026-10-24 by the triage-lane deep sweep: production logs show `remesh-service` segfaulting ("Uncaught signal: 11") twice in 24h inside `_convert_to_fbx_preserving_rig` → `_blender_convert`, the exact "Blender dies in its own interpreter teardown after a fully written export" failure that commit `59d9aa9cd` (2026-08-11, "stop discarding a finished FBX because Blender crashed on the way out") already fixed by judging completion on the FACE_COUNT marker and a non-empty output file rather than the child's exit code. `gcloud run revisions describe remesh-service-00005-8rr --region us-central1` shows that revision was created 2026-07-16, five weeks before the fix landed: this is pure deploy lag on a service three.ws's own deploy cadence does not currently cover (row 1 tracks `three-ws-api`, which is current at `660b5ad97`, 0 commits behind `threews/main`; the 20-service Cloud Run fleet deploys independently per service). No code change needed, the fix already exists; only a rebuild and redeploy of this one GPU worker is owner-gated. Command: `gcloud builds submit --config workers/remesh/cloudbuild.yaml .` from the repo root, then confirm no new "Uncaught signal: 11" lines appear for `remesh-service` over the following 24h. | one build + one deploy | Stops rig-preserving FBX exports from occasionally crashing the whole `remesh-service` instance mid-request | `workers/remesh/main.py` (`_blender_convert`), [docs/ops/production-log-triage.md](../../docs/ops/production-log-triage.md) |
+| 70 | **Set `GITHUB_TOKEN` on `three-ws-api`.** Added 2026-10-24, order 412 close-out. Production runs every `api.github.com` call (ship feed, commit-feed-push cron) unauthenticated, sharing one 60-requests/hour budget per egress IP across both features; `gcloud run services describe three-ws-api --region us-central1 --project aerial-vehicle-466722-p5` confirms no `GITHUB_TOKEN`/`GH_TOKEN` is configured. Order 412 already made exhaustion a non-event (`/api/ship/feed` now serves the last-known-good feed marked stale instead of throwing; `pushTelegramLane()` now returns `{ skipped: 'github_rate_limited' }` instead of throwing), so this is optional, not a blocker. Setting the token raises the shared ceiling from 60 to 5,000 requests/hour, which reduces how often those stale/skip paths are ever exercised. Command: `gcloud run services update three-ws-api --region us-central1 --update-env-vars GITHUB_TOKEN=<a GitHub personal access token with no special scopes; read-only public access is enough>`. | minutes | Fewer stale ship-feed reads and skipped commit-feed-push ticks under shared-IP GitHub traffic | this order's commit, `api/ship/feed.js`, `api/_lib/commit-feed-push.js` |
Deleted 2026-09-01: the former row 5 (`WALLET_ENCRYPTION_KEY_PREVIOUS` policy), which
declared itself "delete on reading" and had outlived that by three weeks.
diff --git a/tests/commit-feed-push.test.js b/tests/commit-feed-push.test.js
index ddd867ec8..fd39303a2 100644
--- a/tests/commit-feed-push.test.js
+++ b/tests/commit-feed-push.test.js
@@ -1,5 +1,18 @@
-import { describe, it, expect } from 'vitest';
-import {
+import { readFileSync } from 'node:fs';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { PGlite } from '@electric-sql/pglite';
+
+const dbState = { pg: null };
+
+vi.mock('../api/_lib/db.js', () => {
+ const sql = async (strings, ...values) => {
+ const text = strings.reduce((acc, part, i) => acc + part + (i < values.length ? `$${i + 1}` : ''), '');
+ return (await dbState.pg.query(text, values)).rows;
+ };
+ return { sql };
+});
+
+const {
commitHeadline,
commitSummary,
commitPreviewUrl,
@@ -7,8 +20,23 @@ import {
commitDate,
fetchCommitsSince,
newCommitsSince,
-} from '../api/_lib/commit-feed-push.js';
-import commitOg from '../api/commit-og.js';
+ githubRateLimitError,
+ isGithubRateLimitError,
+ pushTelegramLane,
+} = await import('../api/_lib/commit-feed-push.js');
+const { default: commitOg } = await import('../api/commit-og.js');
+
+const MIG = new URL('../api/_lib/migrations/', import.meta.url);
+const APP_SETTINGS = (() => {
+ const src = readFileSync(new URL('20260922210000_free_tier_models.sql', MIG), 'utf8');
+ const start = src.indexOf('CREATE TABLE IF NOT EXISTS app_settings');
+ return src.slice(start, src.indexOf(');', start) + 2);
+})();
+
+beforeEach(async () => {
+ dbState.pg = new PGlite();
+ await dbState.pg.exec(APP_SETTINGS);
+});
const commit = (over = {}) => ({
sha: 'abcdef1234567890abcdef1234567890abcdef12',
@@ -197,6 +225,46 @@ describe('newCommitsSince', () => {
});
});
+describe('githubRateLimitError / isGithubRateLimitError', () => {
+ it('round-trips a tagged error as rate-limited', () => {
+ const err = githubRateLimitError(Date.parse('2026-08-14T20:00:00Z'));
+ expect(isGithubRateLimitError(err)).toBe(true);
+ expect(err.retryAfter).toBe('2026-08-14T20:00:00.000Z');
+ expect(err.message).toContain('GitHub rate limit exhausted');
+ });
+
+ it('reads a generic error as not rate-limited', () => {
+ expect(isGithubRateLimitError(new Error('boom'))).toBe(false);
+ expect(isGithubRateLimitError(null)).toBe(false);
+ });
+});
+
+describe('pushTelegramLane', () => {
+ const ORIGINAL_ENV = { ...process.env };
+
+ beforeEach(() => {
+ process.env.TELEGRAM_BOT_TOKEN = 'test-bot-token';
+ process.env.TELEGRAM_COMMITS_CHAT_ID = 'test-chat-id';
+ });
+
+ afterEach(() => {
+ process.env = { ...ORIGINAL_ENV };
+ });
+
+ it('skips gracefully instead of throwing when GitHub answers a rate limit', async () => {
+ const resetMs = Date.parse('2026-08-14T20:00:00Z');
+ const getPage = async () => {
+ throw githubRateLimitError(resetMs);
+ };
+ const result = await pushTelegramLane({ getPage });
+ expect(result).toEqual({
+ posted: 0,
+ skipped: 'github_rate_limited',
+ retryAfter: new Date(resetMs).toISOString(),
+ });
+ });
+});
+
// Minimal ServerResponse stub for the HTML handler.
function capture() {
const res = {
diff --git a/tests/ship-feed-endpoint.test.js b/tests/ship-feed-endpoint.test.js
index d99b133ad..7291932c7 100644
--- a/tests/ship-feed-endpoint.test.js
+++ b/tests/ship-feed-endpoint.test.js
@@ -50,6 +50,17 @@ const fetchUpstream = vi.fn(async () => ({
vi.mock('../api/_lib/upstream-fetch.js', () => ({ fetchUpstream }));
const { default: handler } = await import('../api/ship/feed.js');
+const { cacheDel } = await import('../api/_lib/cache.js');
+
+const RATE_LIMIT_RESPONSE = {
+ ok: false,
+ status: 403,
+ headers: {
+ get: (h) =>
+ h === 'x-ratelimit-remaining' ? '0' : h === 'x-ratelimit-reset' ? String(Math.floor(Date.now() / 1000) + 3600) : null,
+ },
+ json: async () => ({ message: 'API rate limit exceeded' }),
+};
function mkRes() {
return {
@@ -147,6 +158,22 @@ describe('GET /api/ship/feed', () => {
expect(fetchUpstream.mock.calls.length).toBe(callsAfterFirst);
});
+ it('serves the last-good feed, marked stale, when GitHub is rate-limited', async () => {
+ const first = JSON.parse((await call('/api/ship/feed?limit=20')).body);
+ expect(first.stale).toBe(false);
+
+ // Drop the fresh cache entry but leave the last-known-good mirror in
+ // place, so the next call is forced through fn() and has to fall back.
+ await cacheDel('ship:feed:20:internal');
+
+ fetchUpstream.mockImplementationOnce(async () => RATE_LIMIT_RESPONSE);
+ const res = await call('/api/ship/feed?limit=20');
+ expect(res.statusCode).toBe(200);
+ const second = JSON.parse(res.body);
+ expect(second.stale).toBe(true);
+ expect(second.stats.commits).toBe(first.stats.commits);
+ });
+
it('refuses a non-GET method', async () => {
const req = { method: 'POST', url: '/api/ship/feed', headers: {}, socket: { remoteAddress: '127.0.0.1' } };
const res = mkRes();