Every analytics tool wants a subscription. And your data. I built Twiligent instead. YouTube stats. Instagram stats. One dashboard. Runs locally on your machine.
fix(security): bound the daily token refresh so retention always runs
spacesdrive2dca33422ca5
diff --git a/backend/jobs/scheduled.ts b/backend/jobs/scheduled.tsindex 4b2ee88..60ab947 100644--- a/backend/jobs/scheduled.ts+++ b/backend/jobs/scheduled.ts@@ -3,7 +3,7 @@ import { getSupabase } from '../database/supabase.ts';import { getRedis, getRateLimitRedis } from '../database/redis.ts';import { coreSecretsMessage, missingCoreSecrets, readConfig } from '../lib/config.ts';import { FREE_PLAN_SUBREQUESTS } from '../lib/automationDelivery.ts';-import { createLogger } from '../lib/logger.ts';+import { createLogger, errorFields } from '../lib/logger.ts';import type { Services } from '../container.ts';import type { Env } from '../types/env.ts';import type { RequestContext } from '../types/context.ts';@@ -61,6 +61,15 @@ async function dispatch(services: Services, context: RequestContext): Promise<vo}}+// a failed daily step is logged and the next one still runs, so a token refresh problem never stops the retention purge+export async function dailyStep(context: RequestContext, step: string, work: () => Promise<unknown>): Promise<void> {+ try {+ await work();+ } catch (err) {+ context.log.error('daily step failed', { step, ...errorFields(err) });+ }+}+// the cron entry point: the same services the API uses, built from the Worker env instead of a requestexport async function runScheduledJob(cron: string, env: Env, scheduledTime = Date.now()): Promise<void> {const context = cronContext(cron);@@ -83,8 +92,8 @@ export async function runScheduledJob(cron: string, env: Env, scheduledTime = Da} else if (plan === 'reconcile') {await services.automationReconciler.run();} else if (plan === 'daily') {- await services.tokenRefresh.refreshExpiringInstagramTokens();- await services.retention.purge(Date.now(), { commentBatches: RETENTION_BATCHES });+ await dailyStep(context, 'token refresh', () => services.tokenRefresh.refreshExpiringInstagramTokens());+ await dailyStep(context, 'retention purge', () => services.retention.purge(Date.now(), { commentBatches: RETENTION_BATCHES }));} else {context.log.warn('no job for this cron', { cron });return;diff --git a/backend/services/tokenRefreshService.ts b/backend/services/tokenRefreshService.tsindex d31a72a..25ef2ac 100644--- a/backend/services/tokenRefreshService.ts+++ b/backend/services/tokenRefreshService.ts@@ -17,6 +17,13 @@ type ExpiringInstagramAccount = Account & { accessToken: string; tokenExpiresAt:const DEFAULT_TOKEN_LIFETIME_S = 5184000;const REFRESH_WINDOW_MS = 15 * 24 * 60 * 60 * 1000;+// each refresh is one call to Instagram and one database write; ten leave the daily run's retention purge its share of+// the 50 subrequests, and the fifteen day window gives every token many runs to be reached+export const MAX_REFRESHES_PER_RUN = 10;++function expiresAt(account: ExpiringInstagramAccount): number {+ return new Date(account.tokenExpiresAt).getTime();+}function isInstagramWithToken(account: Account): account is ExpiringInstagramAccount {return account.platform === 'instagram' && !!account.accessToken && !!account.tokenExpiresAt;@@ -47,9 +54,18 @@ export function createTokenRefreshService({ accounts, context }: TokenRefreshDeplog.error('token refresh could not load accounts', errorFields(err));return;}- const soon = Date.now() + REFRESH_WINDOW_MS;- const due = listed.filter(isInstagramWithToken).filter(account => new Date(account.tokenExpiresAt).getTime() <= soon);- for (const account of due) await refreshStoredToken(account);+ const now = Date.now();+ const soon = now + REFRESH_WINDOW_MS;+ const withToken = listed.filter(isInstagramWithToken);+ // Instagram refuses to refresh an expired token, so retrying one every day only spends the run; the account+ // needs connecting again+ const expired = withToken.filter(account => expiresAt(account) <= now);+ if (expired.length > 0) log.warn('instagram tokens expired; the accounts need connecting again', { accountIds: expired.map(account => account.id) });+ const due = withToken+ .filter(account => expiresAt(account) > now && expiresAt(account) <= soon)+ .sort((first, second) => expiresAt(first) - expiresAt(second));+ for (const account of due.slice(0, MAX_REFRESHES_PER_RUN)) await refreshStoredToken(account);+ if (due.length > MAX_REFRESHES_PER_RUN) log.info('token refresh left accounts for the next run', { left: due.length - MAX_REFRESHES_PER_RUN });},};}diff --git a/backend/tests/fixtures/tokenRefreshService.refresh.golden.json b/backend/tests/fixtures/tokenRefreshService.refresh.golden.jsonindex fdd2389..ddab3fe 100644--- a/backend/tests/fixtures/tokenRefreshService.refresh.golden.json+++ b/backend/tests/fixtures/tokenRefreshService.refresh.golden.json@@ -7,10 +7,6 @@["error","{\"level\":\"error\",\"message\":\"instagram token refresh failed\",\"requestId\":\"cron-test\",\"accountId\":\"a6\",\"username\":\"failing\",\"error\":\"Session has expired\",\"errorName\":\"UpstreamError\"}"- ],- [- "log",- "{\"level\":\"info\",\"message\":\"instagram token refreshed\",\"requestId\":\"cron-test\",\"accountId\":\"a7\",\"username\":\"expired\",\"tokenExpiresAt\":\"2026-10-31T12:00:00.000Z\"}"]],"requests": [@@ -21,10 +17,6 @@{"url": "https://graph.instagram.com/refresh_access_token?grant_type=ig_refresh_token&access_token=T6","method": "GET"- },- {- "url": "https://graph.instagram.com/refresh_access_token?grant_type=ig_refresh_token&access_token=T7",- "method": "GET"}],"calls": [@@ -51,19 +43,6 @@}},"filters": []- },- {- "type": "rpc",- "table": "merge_account_data",- "values": {- "p_id": "a7",- "p_user_id": null,- "p_patch": {- "accessToken": "N7",- "tokenExpiresAt": "2026-10-31T12:00:00.000Z"- }- },- "filters": []}],"table": [@@ -132,10 +111,9 @@"user_id": "user-2","data": {"username": "expired",- "accessToken": "N7",- "tokenExpiresAt": "2026-10-31T12:00:00.000Z"- },- "updated_at": "2026-09-01T12:00:00.000Z"+ "accessToken": "T7",+ "tokenExpiresAt": "2026-08-30T12:00:00.000Z"+ }}]}diff --git a/backend/tests/jobs/scheduled.test.ts b/backend/tests/jobs/scheduled.test.tsindex 30f0d2a..9c37e7d 100644--- a/backend/tests/jobs/scheduled.test.ts+++ b/backend/tests/jobs/scheduled.test.ts@@ -2,7 +2,8 @@ import { test, mock } from 'node:test';import assert from 'node:assert/strict';import { readFileSync } from 'node:fs';import type { Env } from '../../types/env.ts';-import { DAILY_CRON, EVERY_FIVE_MINUTES_CRON, EVERY_MINUTE_CRON, cronPlan, runScheduledJob } from '../../jobs/scheduled.ts';+import { DAILY_CRON, EVERY_FIVE_MINUTES_CRON, EVERY_MINUTE_CRON, cronPlan, dailyStep, runScheduledJob } from '../../jobs/scheduled.ts';+import { createLogger } from '../../lib/logger.ts';const at = (time: string) => Date.parse(`2026-10-16T${time}:00Z`);@@ -38,3 +39,16 @@ test('a cron run without the Supabase secrets logs which are missing and stops iassert.equal(line.message, 'cron run skipped');assert.match(line.reason, /SUPABASE_SERVICE_KEY, SUPABASE_ANON_KEY Worker secrets are missing/);});++test('a failed daily step is logged and the next step still runs', async t => {+ const errors: string[] = [];+ const spy = mock.method(console, 'error', (line: string) => { errors.push(line); });+ t.after(() => spy.mock.restore());+ const context = { requestId: 'cron-test', signal: new AbortController().signal, log: createLogger({ requestId: 'cron-test' }) };+ const ran: string[] = [];+ await dailyStep(context, 'token refresh', async () => { throw new Error('Instagram is down'); });+ await dailyStep(context, 'retention purge', async () => { ran.push('retention'); });+ assert.deepEqual(ran, ['retention']);+ assert.match(errors[0], /daily step failed/);+ assert.match(errors[0], /token refresh/);+});diff --git a/backend/tests/services/tokenRefreshService.test.ts b/backend/tests/services/tokenRefreshService.test.tsindex 7e6589e..7eb5afd 100644--- a/backend/tests/services/tokenRefreshService.test.ts+++ b/backend/tests/services/tokenRefreshService.test.ts@@ -1,5 +1,6 @@import { describe, test, before, after, afterEach, mock } from 'node:test';-import { createTokenRefreshService } from '../../services/tokenRefreshService.ts';+import assert from 'node:assert/strict';+import { MAX_REFRESHES_PER_RUN, createTokenRefreshService } from '../../services/tokenRefreshService.ts';import { createAccountRepository } from '../../repositories/accountRepository.ts';import { createLogger } from '../../lib/logger.ts';import { assertGolden } from '../fixtures/golden.ts';@@ -66,4 +67,17 @@ describe('refreshExpiringInstagramTokens golden output', () => {await refreshTokens({ from() { throw new Error('database unavailable'); } } as unknown as SupabaseClient);assertGolden({ lines, requests: fetchMock.requests, calls: notDue.calls }, 'tokenRefreshService.idle');});++ test('a run refreshes at most its share of tokens, soonest to expire first', async () => {+ const rows = Array.from({ length: MAX_REFRESHES_PER_RUN + 3 }, (_, i) => accountRow({+ id: `c${i}`, platform: 'instagram', username: `u${i}`, accessToken: `T${i}`, tokenExpiresAt: expiresIn(14 - i),+ }));+ fetchMock = installFetch(() => reply({ access_token: 'NEW', expires_in: 5184000 }));+ captureConsole();+ await refreshTokens(fakeSupabase(rows));+ const refreshed = fetchMock.requests.map(request => new URL(request.url).searchParams.get('access_token'));+ assert.equal(refreshed.length, MAX_REFRESHES_PER_RUN);+ assert.deepEqual(refreshed.slice(0, 3), ['T12', 'T11', 'T10']);+ assert.ok(!refreshed.includes('T0'));+ });});diff --git a/docs/architecture/backend/CRON.md b/docs/architecture/backend/CRON.mdindex 009a11c..f18c69c 100644--- a/docs/architecture/backend/CRON.md+++ b/docs/architecture/backend/CRON.md@@ -42,8 +42,8 @@ export async function runScheduledJob(cron: string, env: Env, scheduledTime = Da} else if (plan === 'reconcile') {await services.automationReconciler.run();} else if (plan === 'daily') {- await services.tokenRefresh.refreshExpiringInstagramTokens();- await services.retention.purge(Date.now(), { commentBatches: RETENTION_BATCHES });+ await dailyStep(context, 'token refresh', () => services.tokenRefresh.refreshExpiringInstagramTokens());+ await dailyStep(context, 'retention purge', () => services.retention.purge(Date.now(), { commentBatches: RETENTION_BATCHES }));}}```@@ -103,11 +103,12 @@ A run that stops is recovered by a later one: a job left `processing` for more t**What it does:**1. Lists the Instagram accounts of every user with `accountRepository.listByPlatform('instagram')`, filtered in SQL, and keeps those with a token and an expiry-2. For each account, checks if the Instagram long-lived token expires within 15 days-3. If expiring soon: calls `refreshLongLivedToken()` (`GET https://graph.instagram.com/refresh_access_token?grant_type=ig_refresh_token&access_token=<token>`)-4. Updates the account's `accessToken` and `tokenExpiresAt` with `accountRepository.updateById()`, which merges them into `data` in one statement (`merge_account_data`), so an analytics refresh at the same moment cannot put the old token back+2. Skips tokens that have already expired, since Instagram refuses to refresh them, and logs their accounts once a day as needing to be connected again (`instagram tokens expired; the accounts need connecting again`)+3. Takes the tokens that expire within the next 15 days, soonest first, at most `MAX_REFRESHES_PER_RUN` (10) a run, so the run keeps subrequests for the retention purge; the rest wait for the next day, which the 15 day window allows, and the log says how many (`left`)+4. For each one: calls `refreshLongLivedToken()` (`GET https://graph.instagram.com/refresh_access_token?grant_type=ig_refresh_token&access_token=<token>`)+5. Updates the account's `accessToken` and `tokenExpiresAt` with `accountRepository.updateById()`, which merges them into `data` in one statement (`merge_account_data`), so an analytics refresh at the same moment cannot put the old token back-One failed account is logged and skipped; it never stops the others. `backend/tests/services/tokenRefreshService.test.ts` covers the selection and the updates.+One failed account is logged and skipped; it never stops the others. Each daily step runs inside `dailyStep()`, which logs a failure (`daily step failed`) and moves on, so a refresh problem never stops the retention purge (audit lead L7). `backend/tests/services/tokenRefreshService.test.ts` covers the selection and the updates.After the refresh, `retention.purge()` in `backend/services/retentionService.ts` deletes received comments older than 90 days (their reply attempts and follow-ups go with them) and audit events older than a year, across all users. Comments are deleted by `purge_automation_comments` in batches of 1,000, oldest first, at most 20 batches a run; a larger backlog is left for the next day and the log says so (`commentsLeft`).