Every analytics tool wants a subscription. And your data. I built Twiligent instead. YouTube stats. Instagram stats. One dashboard. Runs locally on your machine.
fix(security): count YouTube quota in units per member and Pacific day
spacesdrivefa565c2463a8
diff --git a/backend/container.ts b/backend/container.tsindex b3f4e84..b51d9df 100644--- a/backend/container.ts+++ b/backend/container.ts@@ -4,6 +4,8 @@ import { createScheduledPostRepository } from './repositories/scheduledPostReposimport { createTrackedPostRepository } from './repositories/trackedPostRepository.ts';import { createSettingsRepository } from './repositories/settingsRepository.ts';import { createEmailQuotaRepository } from './repositories/emailQuotaRepository.ts';+import { createYouTubeQuotaRepository } from './repositories/youtubeQuotaRepository.ts';+import { createYouTubeQuotaService } from './services/youtubeQuotaService.ts';import { createAuditRepository } from './repositories/auditRepository.ts';import { createAutomationRuleRepository } from './repositories/automationRuleRepository.ts';import { createAutomationCommentRepository } from './repositories/automationCommentRepository.ts';@@ -113,16 +115,18 @@ export function createServices(clients: Clients, config: AppConfig, context: Reqconst mcpTokens = createMcpAccessTokenRepository(clients.supabase);const mcpGrants = createMcpClientGrantRepository(clients.supabase);const scheduledPublisher = createScheduledPublisher({ accounts, posts, context });+ // counters on the fail-fast client the rate limits use: without Redis nothing is metered+ const youtubeQuota = createYouTubeQuotaService({ quota: createYouTubeQuotaRepository(clients.rateLimitRedis) });const services: Services = {- accounts: createAccountService({ accounts, cache, audit, config, context }),- youtubeAnalytics: createYouTubeAnalyticsService({ accounts, cache, config, context }),+ accounts: createAccountService({ accounts, cache, audit, config, context, youtubeQuota }),+ youtubeAnalytics: createYouTubeAnalyticsService({ accounts, cache, config, context, youtubeQuota }),instagramAnalytics: createInstagramAnalyticsService({ accounts, cache, context }),publishing: createPublishingService({ accounts, config, context }),scheduledPosts: createScheduledPostService({ accounts, posts }),scheduledPublisher,tokenRefresh: createTokenRefreshService({ accounts, context }),- trackedContent: createTrackedContentService({ tracked, config, context }),- overview: createOverviewService({ accounts, tracked, cache, config }),+ trackedContent: createTrackedContentService({ tracked, config, context, youtubeQuota }),+ overview: createOverviewService({ accounts, tracked, cache, config, youtubeQuota }),settings: createSettingsService({ settings, config }),instagramConnect: createInstagramConnectService({ accounts, audit, config, context }),authEmail: createAuthEmailService({ quota: createEmailQuotaRepository(clients.rateLimitRedis), config, context }),diff --git a/backend/controllers/accountController.ts b/backend/controllers/accountController.tsindex a7a0ce8..78860a8 100644--- a/backend/controllers/accountController.ts+++ b/backend/controllers/accountController.ts@@ -15,7 +15,7 @@ export const listAccounts = controller.createHandlers(async c => {});export const resolveChannel = controller.createHandlers(jsonBody(parseChannelInput), async c => {- const data = await servicesFor(c).accounts.resolveChannel(c.req.valid('json').input);+ const data = await servicesFor(c).accounts.resolveChannel(c.get('userId'), c.req.valid('json').input);return c.json({ success: true, data } satisfies ResolveChannelResponse);});diff --git a/backend/lib/youtubeQuota.ts b/backend/lib/youtubeQuota.tsnew file mode 100644index 0000000..e42386b--- /dev/null+++ b/backend/lib/youtubeQuota.ts@@ -0,0 +1,19 @@+// the rules for the instance's shared YouTube Data API quota; pure, so tests can check them without Redis++// a Google Cloud project gets 10,000 units a day by default; one member may use a quarter of them, so a single person can+// never spend the allowance every other member's analytics depend on+export const USER_DAILY_UNITS = 2500;++// search.list costs 100 units; every list call Twiligent makes (channels, playlistItems, videos) costs 1+const ENDPOINT_UNITS: Record<string, number> = { search: 100 };++export function unitsFor(endpoint: string): number {+ return ENDPOINT_UNITS[endpoint] ?? 1;+}++// YouTube resets the quota at midnight Pacific time, so the counters are kept per Pacific day+export function pacificDay(now: number = Date.now()): string {+ return new Intl.DateTimeFormat('en-CA', { timeZone: 'America/Los_Angeles', year: 'numeric', month: '2-digit', day: '2-digit' }).format(now);+}++export const QUOTA_USED_MESSAGE = 'You have used your share of this instance\'s YouTube data for today. It resets at midnight Pacific time; cached analytics stay available.';diff --git a/backend/repositories/youtubeQuotaRepository.ts b/backend/repositories/youtubeQuotaRepository.tsnew file mode 100644index 0000000..af577b4--- /dev/null+++ b/backend/repositories/youtubeQuotaRepository.ts@@ -0,0 +1,39 @@+import type { Redis } from '../types/app.ts';++// the counters outlive their Pacific day by a day, so units spent just before midnight are still counted when it rolls+const COUNTER_TTL_SECONDS = 2 * 24 * 60 * 60;++// units a member spent on the YouTube Data API on one Pacific day; without Redis, or when it does not answer, nothing is+// counted and nothing is refused, as with the rest of the cache+export interface YouTubeQuotaRepository {+ usedForUser(userId: string, day: string): Promise<number | null>;+ addForUser(userId: string, day: string, units: number): Promise<void>;+}++function key(userId: string, day: string): string {+ return `ytunits:${day}:${userId}`;+}++export function createYouTubeQuotaRepository(redis: Redis | null): YouTubeQuotaRepository {+ return {+ async usedForUser(userId, day) {+ if (!redis) return null;+ try {+ return Number(await redis.get<number | string>(key(userId, day)) ?? 0);+ } catch {+ return null;+ }+ },++ async addForUser(userId, day, units) {+ if (!redis || units <= 0) return;+ try {+ const counter = key(userId, day);+ // one request for both commands: each request is a subrequest the invocation pays for+ await redis.pipeline().incrby(counter, units).expire(counter, COUNTER_TTL_SECONDS).exec();+ } catch {+ // a lost count only makes the limit more generous for one day+ }+ },+ };+}diff --git a/backend/services/accountService.ts b/backend/services/accountService.tsindex bd9ae55..65724da 100644--- a/backend/services/accountService.ts+++ b/backend/services/accountService.ts@@ -1,7 +1,10 @@import { resolveChannelId, fetchChannelData } from './youtube/youtubeChannels.ts';import { fetchInstagramProfile } from './instagram/instagramApi.ts';import { buildYouTubeAccount } from './accountFactory.ts';-import { ConflictError, NotConfiguredError, NotFoundError, ValidationError, isUniqueViolation } from '../lib/errors.ts';+import { ConflictError, NotConfiguredError, NotFoundError, RateLimitedError, ValidationError, isUniqueViolation } from '../lib/errors.ts';+import { UNMETERED_YOUTUBE_QUOTA } from './youtubeQuotaService.ts';+import type { YouTubeQuotaService } from './youtubeQuotaService.ts';+import type { YouTubeKey } from './youtube/youtubeApi.ts';import type { AccountRepository } from '../repositories/accountRepository.ts';import type { AnalyticsCacheRepository } from '../repositories/analyticsCacheRepository.ts';import type { AuditRepository } from '../repositories/auditRepository.ts';@@ -17,6 +20,7 @@ export interface AccountServiceDeps {audit: AuditRepository;config: AppConfig;context: RequestContext;+ youtubeQuota?: YouTubeQuotaService;}export interface RefreshAllOutcome {@@ -26,7 +30,7 @@ export interface RefreshAllOutcome {export interface AccountService {list(userId: string): Promise<Account[]>;- resolveChannel(input: string): Promise<YouTubeChannelData>;+ resolveChannel(userId: string, input: string): Promise<YouTubeChannelData>;addYouTube(userId: string, input: string): Promise<NewAccount>;refresh(userId: string, accountId: string): Promise<Account>;refreshAll(userId: string): Promise<RefreshAllOutcome>;@@ -67,7 +71,7 @@ function youtubeUpdates(channelData: YouTubeChannelData, full: boolean): Partialconst CHANNEL_ADDED = 'This channel is already added';-export function createAccountService({ accounts, cache, audit, config, context }: AccountServiceDeps): AccountService {+export function createAccountService({ accounts, cache, audit, config, context, youtubeQuota = UNMETERED_YOUTUBE_QUOTA }: AccountServiceDeps): AccountService {const { signal } = context;function youtubeKey(): string {@@ -92,35 +96,33 @@ export function createAccountService({ accounts, cache, audit, config, context }}// the platform call needs these fields; a missing one is reported before any request goes out- async function fetchProfileUpdates(account: Account, full: boolean): Promise<Partial<Account>> {+ async function fetchProfileUpdates(account: Account, full: boolean, key?: YouTubeKey): Promise<Partial<Account>> {if (account.platform === 'instagram') {if (!account.accessToken) throw new ValidationError(NO_TOKEN_MESSAGE);return instagramUpdates(await fetchInstagramProfile(account.igUserId, account.accessToken, signal), full);}if (!account.channelId) throw new ValidationError(NO_CHANNEL_MESSAGE);- return youtubeUpdates(await fetchChannelData(account.channelId, youtubeKey(), signal), full);+ return youtubeUpdates(await fetchChannelData(account.channelId, key ?? youtubeKey(), signal), full);}- async function refreshListed(userId: string, account: Account): Promise<void> {- const updates = { lastUpdated: new Date().toISOString(), ...await fetchProfileUpdates(account, false) };+ async function refreshListed(userId: string, account: Account, key?: YouTubeKey): Promise<void> {+ const updates = { lastUpdated: new Date().toISOString(), ...await fetchProfileUpdates(account, false, key) };await accounts.updateForUser(account.id, updates, userId);}return {list: userId => accounts.listForUser(userId),- async resolveChannel(input) {- const apiKey = youtubeKey();- return fetchChannelData(await resolveChannelId(input, apiKey, signal), apiKey, signal);+ // a lookup by name can cost 100 quota units, so it counts against the member who asked+ async resolveChannel(userId, input) {+ return youtubeQuota.run(userId, youtubeKey(), async key => fetchChannelData(await resolveChannelId(input, key, signal), key, signal));},async addYouTube(userId, input) {- const apiKey = youtubeKey();- const channelId = await resolveChannelId(input, apiKey, signal);- const [channelData, existing] = await Promise.all([- fetchChannelData(channelId, apiKey, signal),- accounts.listForUser(userId),- ]);+ const [channelId, channelData, existing] = await youtubeQuota.run(userId, youtubeKey(), async key => {+ const id = await resolveChannelId(input, key, signal);+ return [id, ...await Promise.all([fetchChannelData(id, key, signal), accounts.listForUser(userId)])] as const;+ });if (existing.find(account => account.channelId === channelId)) throw new ConflictError(CHANNEL_ADDED);const account = buildYouTubeAccount(channelData);@@ -130,7 +132,10 @@ export function createAccountService({ accounts, cache, audit, config, context }async refresh(userId, accountId) {const account = await findOwned(userId, accountId);- const updates = { lastUpdated: new Date().toISOString(), ...await fetchProfileUpdates(account, true) };+ const fetched = account.platform === 'youtube'+ ? await youtubeQuota.run(userId, youtubeKey(), key => fetchProfileUpdates(account, true, key))+ : await fetchProfileUpdates(account, true);+ const updates = { lastUpdated: new Date().toISOString(), ...fetched };await accounts.updateForUser(account.id, updates, userId);return { ...account, ...updates };},@@ -138,7 +143,20 @@ export function createAccountService({ accounts, cache, audit, config, context }// one failure must not stop the rest; results line up with the accounts by indexasync refreshAll(userId) {const listed = await accounts.listForUser(userId);- const settled = await Promise.allSettled(listed.map(account => refreshListed(userId, account)));+ const settle = (key?: YouTubeKey) => Promise.allSettled(listed.map(account => refreshListed(userId, account, key)));+ let settled: PromiseSettledResult<void>[];+ if (!config.youtubeApiKey || !listed.some(account => account.platform === 'youtube')) {+ settled = await settle();+ } else {+ // one metered run for every channel, so a member with many channels pays two quota subrequests, not two each+ try {+ settled = await youtubeQuota.run(userId, config.youtubeApiKey, key => settle(key));+ } catch (err) {+ if (!(err instanceof RateLimitedError)) throw err;+ // a used-up YouTube share stops only the channels; Instagram accounts still refresh+ settled = await Promise.allSettled(listed.map(account => (account.platform === 'youtube' ? Promise.reject(err) : refreshListed(userId, account))));+ }+ }const results: AccountRefreshResult[] = settled.map((outcome, i) => (outcome.status === 'fulfilled'? { id: listed[i].id, success: true }: { id: listed[i].id, success: false, error: (outcome.reason as Error)?.message }));diff --git a/backend/services/overviewService.ts b/backend/services/overviewService.tsindex 394bfec..fb9289b 100644--- a/backend/services/overviewService.ts+++ b/backend/services/overviewService.ts@@ -1,6 +1,8 @@import { fetchAllVideos } from './youtube/youtubeVideos.ts';import { fetchInstagramMedia } from './instagram/instagramMedia.ts';import { summarizeContentTypes } from './instagram/instagramContentTypes.ts';+import { UNMETERED_YOUTUBE_QUOTA } from './youtubeQuotaService.ts';+import type { YouTubeQuotaService } from './youtubeQuotaService.ts';import type { AccountRepository } from '../repositories/accountRepository.ts';import type { AnalyticsCacheRepository } from '../repositories/analyticsCacheRepository.ts';import type { TrackedPostRepository } from '../repositories/trackedPostRepository.ts';@@ -15,6 +17,7 @@ export interface OverviewDeps {tracked: TrackedPostRepository;cache: AnalyticsCacheRepository;config: AppConfig;+ youtubeQuota?: YouTubeQuotaService;}export interface OverviewSummary {@@ -37,7 +40,7 @@ function sumBy<T>(items: T[], pick: (item: T) => number): number {return items.reduce((total, item) => total + pick(item), 0);}-export function createOverviewService({ accounts, tracked, cache, config }: OverviewDeps): OverviewService {+export function createOverviewService({ accounts, tracked, cache, config, youtubeQuota = UNMETERED_YOUTUBE_QUOTA }: OverviewDeps): OverviewService {function instagramWarmTask(userId: string, account: Account): WarmTask | null {const { igUserId, accessToken } = account;if (!igUserId || !accessToken) return null;@@ -51,10 +54,10 @@ export function createOverviewService({ accounts, tracked, cache, config }: Overconst apiKey = config.youtubeApiKey;const { uploadsPlaylistId } = account;if (!apiKey || !uploadsPlaylistId) return null;- return async signal => {- const videos = await fetchAllVideos(uploadsPlaylistId, apiKey, 10, signal);+ return signal => youtubeQuota.run(userId, apiKey, async key => {+ const videos = await fetchAllVideos(uploadsPlaylistId, key, 10, signal);await cache.setVideos(userId, account.id, { videos, fetchedAt: new Date().toISOString() });- };+ });}return {diff --git a/backend/services/trackedContentService.ts b/backend/services/trackedContentService.tsindex 5fb6d4d..8254bb8 100644--- a/backend/services/trackedContentService.ts+++ b/backend/services/trackedContentService.ts@@ -2,6 +2,8 @@ import { fetchTrackedYouTubeData } from './youtube/youtubeTracked.ts';import { detectContentType, UNSUPPORTED_LINK_MESSAGE } from './trackedContentRules.ts';import { NotConfiguredError, NotFoundError, ValidationError } from '../lib/errors.ts';import { newRowId } from '../lib/ids.ts';+import { UNMETERED_YOUTUBE_QUOTA } from './youtubeQuotaService.ts';+import type { YouTubeQuotaService } from './youtubeQuotaService.ts';import type { TrackedPostRepository, TrackedPostUpdates } from '../repositories/trackedPostRepository.ts';import type { AppConfig } from '../lib/config.ts';import type { RequestContext } from '../types/context.ts';@@ -11,6 +13,7 @@ export interface TrackedContentDeps {tracked: TrackedPostRepository;config: AppConfig;context: RequestContext;+ youtubeQuota?: YouTubeQuotaService;}export interface NewTrackedContent {@@ -32,7 +35,7 @@ export interface TrackedContentService {refresh(userId: string, id: string): Promise<TrackedPost | null>;}-export function createTrackedContentService({ tracked, config, context }: TrackedContentDeps): TrackedContentService {+export function createTrackedContentService({ tracked, config, context, youtubeQuota = UNMETERED_YOUTUBE_QUOTA }: TrackedContentDeps): TrackedContentService {const { signal } = context;function youtubeKey(): string {@@ -46,8 +49,8 @@ export function createTrackedContentService({ tracked, config, context }: Trackereturn item;}- async function fetchVideoData(url: string): Promise<Record<string, unknown>> {- const data = await fetchTrackedYouTubeData(url, youtubeKey(), signal);+ async function fetchVideoData(userId: string, url: string): Promise<Record<string, unknown>> {+ const data = await youtubeQuota.run(userId, youtubeKey(), key => fetchTrackedYouTubeData(url, key, signal));return data as unknown as Record<string, unknown>;}@@ -56,7 +59,7 @@ export function createTrackedContentService({ tracked, config, context }: Trackeasync add(userId, { url, label, category }) {if (!detectContentType(url)) throw new ValidationError(UNSUPPORTED_LINK_MESSAGE);- const data = await fetchVideoData(url);+ const data = await fetchVideoData(userId, url);const id = newRowId();await tracked.create({ id, url, label, category, data }, userId);@@ -79,7 +82,7 @@ export function createTrackedContentService({ tracked, config, context }: Trackeasync refresh(userId, id) {const existing = await findOwned(userId, id);- await tracked.patchDataForUser(id, await fetchVideoData(existing.url), userId);+ await tracked.patchDataForUser(id, await fetchVideoData(userId, existing.url), userId);return tracked.findForUser(id, userId);},};diff --git a/backend/services/youtube/youtubeApi.ts b/backend/services/youtube/youtubeApi.tsindex 5c3cf54..1f11cf6 100644--- a/backend/services/youtube/youtubeApi.ts+++ b/backend/services/youtube/youtubeApi.ts@@ -1,5 +1,6 @@import { createCircuitBreaker } from '../../lib/circuitBreaker.ts';import { UpstreamError, UpstreamTimeoutError, isAbortError } from '../../lib/errors.ts';+import { unitsFor } from '../../lib/youtubeQuota.ts';export type YouTubeApiParams = Record<string, string>;@@ -90,9 +91,20 @@ export interface YouTubeApiVideo {export const youtubeCircuit = createCircuitBreaker('YouTube');-// adds the key to the caller's params object, as callers always pass a fresh one-export function ytFetch<T>(endpoint: string, params: YouTubeApiParams, apiKey: string, signal?: AbortSignal): Promise<T> {- return youtubeCircuit.run(() => requestYouTube<T>(endpoint, params, apiKey, signal), signal);+// the operator's API key, with the meter that counts the quota units each call spends for the member it serves+export interface MeteredYouTubeKey {+ key: string;+ spend(units: number): void;+}++export type YouTubeKey = string | MeteredYouTubeKey;++// adds the key to the caller's params object, as callers always pass a fresh one; YouTube charges a call's units even when+// it fails, so they are counted before it goes out+export function ytFetch<T>(endpoint: string, params: YouTubeApiParams, apiKey: YouTubeKey, signal?: AbortSignal): Promise<T> {+ if (typeof apiKey !== 'string') apiKey.spend(unitsFor(endpoint));+ const key = typeof apiKey === 'string' ? apiKey : apiKey.key;+ return youtubeCircuit.run(() => requestYouTube<T>(endpoint, params, key, signal), signal);}async function requestYouTube<T>(endpoint: string, params: YouTubeApiParams, apiKey: string, signal?: AbortSignal): Promise<T> {diff --git a/backend/services/youtube/youtubeChannels.ts b/backend/services/youtube/youtubeChannels.tsindex fda83ee..3bad4f9 100644--- a/backend/services/youtube/youtubeChannels.ts+++ b/backend/services/youtube/youtubeChannels.ts@@ -1,4 +1,5 @@import { ytFetch } from './youtubeApi.ts';+import type { YouTubeKey } from './youtubeApi.ts';import { NotFoundError, UpstreamTimeoutError, ValidationError } from '../../lib/errors.ts';import type { YouTubeApiParams, YouTubeApiListResponse, YouTubeApiChannel, YouTubeApiSearchResult } from './youtubeApi.ts';import type { YouTubeChannelData } from '../../../shared/types/youtube.ts';@@ -25,7 +26,7 @@ function extractChannelId(input: string | null | undefined): string | null {// null when the lookup finds nothing or fails, so the next strategy runs; a timeout stops the searchasync function lookupChannelId<TItem>(- endpoint: string, params: YouTubeApiParams, pickId: (item: TItem) => string, apiKey: string, signal?: AbortSignal,+ endpoint: string, params: YouTubeApiParams, pickId: (item: TItem) => string, apiKey: YouTubeKey, signal?: AbortSignal,): Promise<string | null> {try {const data = await ytFetch<YouTubeApiListResponse<TItem>>(endpoint, params, apiKey, signal);@@ -37,7 +38,7 @@ async function lookupChannelId<TItem>(}}-export async function resolveChannelId(input: string | null | undefined, apiKey: string, signal?: AbortSignal): Promise<string> {+export async function resolveChannelId(input: string | null | undefined, apiKey: YouTubeKey, signal?: AbortSignal): Promise<string> {const extracted = extractChannelId(input);if (!extracted) throw new ValidationError('Invalid input');if (CHANNEL_ID_PATTERN.test(extracted)) return extracted;@@ -56,7 +57,7 @@ export async function resolveChannelId(input: string | null | undefined, apiKey:throw new NotFoundError('Could not resolve channel. Please provide a valid Channel ID (starts with UC).');}-export async function fetchChannelData(channelId: string, apiKey: string, signal?: AbortSignal): Promise<YouTubeChannelData> {+export async function fetchChannelData(channelId: string, apiKey: YouTubeKey, signal?: AbortSignal): Promise<YouTubeChannelData> {const data = await ytFetch<YouTubeApiListResponse<YouTubeApiChannel>>('channels', {part: 'snippet,statistics,contentDetails,brandingSettings,status,topicDetails',id: channelId,diff --git a/backend/services/youtube/youtubeTracked.ts b/backend/services/youtube/youtubeTracked.tsindex ddd1a7b..68d6039 100644--- a/backend/services/youtube/youtubeTracked.ts+++ b/backend/services/youtube/youtubeTracked.ts@@ -1,4 +1,5 @@import { ytFetch, optionalCount } from './youtubeApi.ts';+import type { YouTubeKey } from './youtubeApi.ts';import { NotFoundError, ValidationError } from '../../lib/errors.ts';import type { YouTubeApiListResponse, YouTubeApiVideo } from './youtubeApi.ts';import type { YouTubeTrackedVideoData } from '../../../shared/types/youtube.ts';@@ -22,7 +23,7 @@ export function extractYouTubeVideoId(url: string): string | null {return searchParams.get('v');}-export async function fetchTrackedYouTubeData(videoUrl: string, apiKey: string, signal?: AbortSignal): Promise<YouTubeTrackedVideoData> {+export async function fetchTrackedYouTubeData(videoUrl: string, apiKey: YouTubeKey, signal?: AbortSignal): Promise<YouTubeTrackedVideoData> {const videoId = extractYouTubeVideoId(videoUrl);if (!videoId) throw new ValidationError('Invalid YouTube URL - could not extract video ID');diff --git a/backend/services/youtube/youtubeVideos.ts b/backend/services/youtube/youtubeVideos.tsindex 4835c75..c2473af 100644--- a/backend/services/youtube/youtubeVideos.ts+++ b/backend/services/youtube/youtubeVideos.ts@@ -1,4 +1,5 @@import { ytFetch, optionalCount } from './youtubeApi.ts';+import type { YouTubeKey } from './youtubeApi.ts';import { UpstreamError } from '../../lib/errors.ts';import type { YouTubeApiListResponse, YouTubeApiParams, YouTubeApiPlaylistItem, YouTubeApiVideo } from './youtubeApi.ts';import type { YouTubeVideo } from '../../../shared/types/youtube.ts';@@ -37,7 +38,7 @@ export function normalizeYouTubeVideo(video: YouTubeApiVideo): YouTubeVideo {}async function fetchPlaylistPage(- playlistId: string, pageToken: string, apiKey: string, signal?: AbortSignal,+ playlistId: string, pageToken: string, apiKey: YouTubeKey, signal?: AbortSignal,): Promise<YouTubeApiListResponse<YouTubeApiPlaylistItem> | null> {const params: YouTubeApiParams = { part: 'snippet,contentDetails', playlistId, maxResults: '50' };if (pageToken) params.pageToken = pageToken;@@ -50,7 +51,7 @@ async function fetchPlaylistPage(}}-async function fetchVideoDetails(playlistItems: YouTubeApiPlaylistItem[], apiKey: string, signal?: AbortSignal): Promise<YouTubeVideo[]> {+async function fetchVideoDetails(playlistItems: YouTubeApiPlaylistItem[], apiKey: YouTubeKey, signal?: AbortSignal): Promise<YouTubeVideo[]> {const videoIds = playlistItems.map(item => item.snippet.resourceId.videoId).join(',');const statsData = await ytFetch<YouTubeApiListResponse<YouTubeApiVideo>>('videos', {part: 'statistics,contentDetails,snippet,status',@@ -59,7 +60,7 @@ async function fetchVideoDetails(playlistItems: YouTubeApiPlaylistItem[], apiKeyreturn statsData.items ? statsData.items.map(normalizeYouTubeVideo) : [];}-export async function fetchAllVideos(playlistId: string, apiKey: string, maxPages = 10, signal?: AbortSignal): Promise<YouTubeVideo[]> {+export async function fetchAllVideos(playlistId: string, apiKey: YouTubeKey, maxPages = 10, signal?: AbortSignal): Promise<YouTubeVideo[]> {const allVideos: YouTubeVideo[] = [];let pageToken = '';let page = 0;diff --git a/backend/services/youtubeAnalyticsService.ts b/backend/services/youtubeAnalyticsService.tsindex 2ac66e5..c238eb6 100644--- a/backend/services/youtubeAnalyticsService.ts+++ b/backend/services/youtubeAnalyticsService.ts@@ -6,9 +6,12 @@ import { NO_YOUTUBE_KEY_MESSAGE } from './accountService.ts';import { NotConfiguredError, NotFoundError, ValidationError } from '../lib/errors.ts';import { errorFields } from '../lib/logger.ts';import { safeAccount } from '../lib/safeAccount.ts';+import { UNMETERED_YOUTUBE_QUOTA } from './youtubeQuotaService.ts';import type { AccountRepository } from '../repositories/accountRepository.ts';import type { AnalyticsCacheRepository } from '../repositories/analyticsCacheRepository.ts';import type { AppConfig } from '../lib/config.ts';+import type { YouTubeQuotaService } from './youtubeQuotaService.ts';+import type { YouTubeKey } from './youtube/youtubeApi.ts';import type { RequestContext } from '../types/context.ts';import type { Account, SafeAccount } from '../../shared/types/domain.ts';import type {@@ -20,6 +23,7 @@ export interface YouTubeAnalyticsDeps {cache: AnalyticsCacheRepository;config: AppConfig;context: RequestContext;+ youtubeQuota?: YouTubeQuotaService;}export interface ChannelAnalytics {@@ -42,7 +46,7 @@ export interface YouTubeAnalyticsService {const MAX_PLAYLIST_PAGES = 10;const NO_PLAYLIST_MESSAGE = 'No uploads playlist stored - delete this channel and add it again';-export function createYouTubeAnalyticsService({ accounts, cache, config, context }: YouTubeAnalyticsDeps): YouTubeAnalyticsService {+export function createYouTubeAnalyticsService({ accounts, cache, config, context, youtubeQuota = UNMETERED_YOUTUBE_QUOTA }: YouTubeAnalyticsDeps): YouTubeAnalyticsService {const { signal, log } = context;function youtubeKey(): string {@@ -57,7 +61,7 @@ export function createYouTubeAnalyticsService({ accounts, cache, config, contextreturn { ...account, uploadsPlaylistId: account.uploadsPlaylistId };}- async function fetchAndCacheVideos(userId: string, accountId: string, playlistId: string, apiKey: string): Promise<ChannelVideos> {+ async function fetchAndCacheVideos(userId: string, accountId: string, playlistId: string, apiKey: YouTubeKey): Promise<ChannelVideos> {const videos = await fetchAllVideos(playlistId, apiKey, MAX_PLAYLIST_PAGES, signal);const fetchedAt = new Date().toISOString();await cache.setVideos(userId, accountId, { videos, fetchedAt });@@ -65,7 +69,7 @@ export function createYouTubeAnalyticsService({ accounts, cache, config, context}// fills creation dates on older accounts once; a failure only skips the extras- async function backfillChannelData(userId: string, account: Account, apiKey: string): Promise<Account> {+ async function backfillChannelData(userId: string, account: Account, apiKey: YouTubeKey): Promise<Account> {if (!account.channelId) return account;try {const channelData = await fetchChannelData(account.channelId, apiKey, signal);@@ -92,10 +96,12 @@ export function createYouTubeAnalyticsService({ accounts, cache, config, contextasync channelAnalytics(userId, accountId) {const apiKey = youtubeKey();const account = await findChannel(userId, accountId);- const { videos } = await fetchAndCacheVideos(userId, accountId, account.uploadsPlaylistId, apiKey);- const analytics = computeVideoAnalytics(videos);- const channel = account.publishedAt ? account : await backfillChannelData(userId, account, apiKey);- return { channel: { ...safeAccount(channel), ...channelMetrics(channel) }, analytics, videoCount: videos.length };+ return youtubeQuota.run(userId, apiKey, async key => {+ const { videos } = await fetchAndCacheVideos(userId, accountId, account.uploadsPlaylistId, key);+ const analytics = computeVideoAnalytics(videos);+ const channel = account.publishedAt ? account : await backfillChannelData(userId, account, key);+ return { channel: { ...safeAccount(channel), ...channelMetrics(channel) }, analytics, videoCount: videos.length };+ });},// served from the cache when it has the channel, without spending quota, unless a person asked for fresh data@@ -106,7 +112,7 @@ export function createYouTubeAnalyticsService({ accounts, cache, config, context}const apiKey = youtubeKey();const account = await findChannel(userId, accountId);- return fetchAndCacheVideos(userId, accountId, account.uploadsPlaylistId, apiKey);+ return youtubeQuota.run(userId, apiKey, key => fetchAndCacheVideos(userId, accountId, account.uploadsPlaylistId, key));},// only channels have the counts being ranked, and stored secrets never leave the Workerdiff --git a/backend/services/youtubeQuotaService.ts b/backend/services/youtubeQuotaService.tsnew file mode 100644index 0000000..3249391--- /dev/null+++ b/backend/services/youtubeQuotaService.ts@@ -0,0 +1,37 @@+import { RateLimitedError } from '../lib/errors.ts';+import { QUOTA_USED_MESSAGE, USER_DAILY_UNITS, pacificDay } from '../lib/youtubeQuota.ts';+import type { YouTubeQuotaRepository } from '../repositories/youtubeQuotaRepository.ts';+import type { MeteredYouTubeKey } from './youtube/youtubeApi.ts';++export interface YouTubeQuotaDeps {+ quota: YouTubeQuotaRepository;+ now?: () => number;+}++export interface YouTubeQuotaService {+ // runs one YouTube operation for a member: refused once they used their daily share, and its units are recorded after+ run<T>(userId: string, apiKey: string, work: (key: MeteredYouTubeKey) => Promise<T>): Promise<T>;+}++// the instance shares one YouTube Data API quota; this keeps one member from spending it for everyone (audit lead L6)+export function createYouTubeQuotaService({ quota, now = Date.now }: YouTubeQuotaDeps): YouTubeQuotaService {+ return {+ async run(userId, apiKey, work) {+ const day = pacificDay(now());+ const used = await quota.usedForUser(userId, day);+ if (used !== null && used >= USER_DAILY_UNITS) throw new RateLimitedError(QUOTA_USED_MESSAGE);+ let spent = 0;+ try {+ return await work({ key: apiKey, spend: units => { spent += units; } });+ } finally {+ // one write per operation, however many calls it made, so metering costs at most two subrequests+ await quota.addForUser(userId, day, spent);+ }+ },+ };+}++// for code paths that have no member to charge, such as tests and tools that do not touch YouTube+export const UNMETERED_YOUTUBE_QUOTA: YouTubeQuotaService = {+ run: (_userId, apiKey, work) => work({ key: apiKey, spend: () => undefined }),+};diff --git a/backend/tests/fixtures/overviewRoutes.warming.golden.json b/backend/tests/fixtures/overviewRoutes.warming.golden.jsonindex f8b4904..940979b 100644--- a/backend/tests/fixtures/overviewRoutes.warming.golden.json+++ b/backend/tests/fixtures/overviewRoutes.warming.golden.json@@ -20,35 +20,35 @@],"requests": [{- "url": "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet%2CcontentDetails&playlistId=UU1&maxResults=50&key=YTKEY",+ "url": "https://graph.instagram.com/v25.0/IG1/media?access_token=T1&limit=50&fields=id%2Ccaption%2Cmedia_type%2Cmedia_product_type%2Cmedia_url%2Cthumbnail_url%2Ccover_url%2Cpermalink%2Ctimestamp%2Clike_count%2Ccomments_count%2Cinsights.metric%28views%29","method": "GET"},{- "url": "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet%2CcontentDetails&playlistId=UU2&maxResults=50&key=YTKEY",+ "url": "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet%2CcontentDetails&playlistId=UU1&maxResults=50&key=YTKEY","method": "GET"},{- "url": "https://graph.instagram.com/v25.0/IG1/media?access_token=T1&limit=50&fields=id%2Ccaption%2Cmedia_type%2Cmedia_product_type%2Cmedia_url%2Cthumbnail_url%2Ccover_url%2Cpermalink%2Ctimestamp%2Clike_count%2Ccomments_count%2Cinsights.metric%28views%29",+ "url": "https://www.googleapis.com/youtube/v3/playlistItems?part=snippet%2CcontentDetails&playlistId=UU2&maxResults=50&key=YTKEY","method": "GET"}],"redisWrites": [{- "key": "videos:user-1:yt1",+ "key": "ig:user-1:ig1","options": {"ex": 21600},"version": 2},{- "key": "videos:user-1:yt2",+ "key": "videos:user-1:yt1","options": {"ex": 21600},"version": 2},{- "key": "ig:user-1:ig1",+ "key": "videos:user-1:yt2","options": {"ex": 21600},diff --git a/backend/tests/lib/youtubeQuota.test.ts b/backend/tests/lib/youtubeQuota.test.tsnew file mode 100644index 0000000..c008d43--- /dev/null+++ b/backend/tests/lib/youtubeQuota.test.ts@@ -0,0 +1,18 @@+import { test } from 'node:test';+import assert from 'node:assert/strict';+import { USER_DAILY_UNITS, pacificDay, unitsFor } from '../../lib/youtubeQuota.ts';++test('a search costs 100 units and every list call Twiligent makes costs 1', () => {+ assert.equal(unitsFor('search'), 100);+ for (const endpoint of ['channels', 'playlistItems', 'videos']) assert.equal(unitsFor(endpoint), 1, endpoint);+});++test('the counters follow the Pacific day YouTube resets on, not the UTC day', () => {+ // 07:30 UTC is still the evening before in California+ assert.equal(pacificDay(Date.parse('2026-11-05T07:30:00Z')), '2026-11-04');+ assert.equal(pacificDay(Date.parse('2026-11-05T09:00:00Z')), '2026-11-05');+});++test('one member may spend a quarter of the default daily quota', () => {+ assert.equal(USER_DAILY_UNITS, 2500);+});diff --git a/backend/tests/repositories/youtubeQuotaRepository.test.ts b/backend/tests/repositories/youtubeQuotaRepository.test.tsnew file mode 100644index 0000000..c8db774--- /dev/null+++ b/backend/tests/repositories/youtubeQuotaRepository.test.ts@@ -0,0 +1,52 @@+import { test } from 'node:test';+import assert from 'node:assert/strict';+import { createYouTubeQuotaRepository } from '../../repositories/youtubeQuotaRepository.ts';+import type { Redis } from '../../types/app.ts';++// the smallest Redis the repository uses: get, and a pipeline of incrby and expire sent as one request+function fakeRedis(store: Record<string, number>, failing = false) {+ const requests: string[][] = [];+ const redis = {+ async get(key: string) {+ if (failing) throw new Error('redis down');+ return store[key] ?? null;+ },+ pipeline() {+ const commands: string[][] = [];+ const chain = {+ incrby(key: string, units: number) { commands.push(['incrby', key, String(units)]); return chain; },+ expire(key: string, seconds: number) { commands.push(['expire', key, String(seconds)]); return chain; },+ async exec() {+ if (failing) throw new Error('redis down');+ requests.push(commands.flat());+ for (const [command, key, value] of commands) if (command === 'incrby') store[key] = (store[key] ?? 0) + Number(value);+ return [];+ },+ };+ return chain;+ },+ };+ return { redis: redis as unknown as Redis, requests };+}++test('units add up per member and Pacific day, in one request per write', async () => {+ const store: Record<string, number> = {};+ const { redis, requests } = fakeRedis(store);+ const repo = createYouTubeQuotaRepository(redis);+ await repo.addForUser('user-1', '2026-11-05', 101);+ await repo.addForUser('user-1', '2026-11-05', 2);+ await repo.addForUser('user-1', '2026-11-05', 0);+ assert.equal(await repo.usedForUser('user-1', '2026-11-05'), 103);+ assert.equal(await repo.usedForUser('user-2', '2026-11-05'), 0);+ assert.equal(requests.length, 2);+ assert.deepEqual(requests[0], ['incrby', 'ytunits:2026-11-05:user-1', '101', 'expire', 'ytunits:2026-11-05:user-1', '172800']);+});++test('without Redis, or when it fails, nothing is counted and nothing throws', async () => {+ const none = createYouTubeQuotaRepository(null);+ assert.equal(await none.usedForUser('user-1', '2026-11-05'), null);+ await none.addForUser('user-1', '2026-11-05', 5);+ const broken = createYouTubeQuotaRepository(fakeRedis({}, true).redis);+ assert.equal(await broken.usedForUser('user-1', '2026-11-05'), null);+ await broken.addForUser('user-1', '2026-11-05', 5);+});diff --git a/backend/tests/services/youtubeQuotaService.test.ts b/backend/tests/services/youtubeQuotaService.test.tsnew file mode 100644index 0000000..d4625a1--- /dev/null+++ b/backend/tests/services/youtubeQuotaService.test.ts@@ -0,0 +1,56 @@+import { test } from 'node:test';+import assert from 'node:assert/strict';+import { createYouTubeQuotaService } from '../../services/youtubeQuotaService.ts';+import { RateLimitedError } from '../../lib/errors.ts';+import { USER_DAILY_UNITS } from '../../lib/youtubeQuota.ts';+import type { YouTubeQuotaRepository } from '../../repositories/youtubeQuotaRepository.ts';++const NOW = Date.parse('2026-11-05T18:00:00Z');++function counters(used: number | null) {+ const added: { userId: string; day: string; units: number }[] = [];+ const quota: YouTubeQuotaRepository = {+ usedForUser: async () => used,+ addForUser: async (userId, day, units) => { added.push({ userId, day, units }); },+ };+ return { quota, added };+}++test('an operation spends its calls\' units on the member and records them once at the end', async () => {+ const { quota, added } = counters(10);+ const service = createYouTubeQuotaService({ quota, now: () => NOW });+ const result = await service.run('user-1', 'KEY', async key => {+ assert.equal(key.key, 'KEY');+ key.spend(1);+ key.spend(100);+ return 'done';+ });+ assert.equal(result, 'done');+ assert.deepEqual(added, [{ userId: 'user-1', day: '2026-11-05', units: 101 }]);+});++test('a member who used their daily share is refused before any call, with a message that says when it resets', async () => {+ const { quota, added } = counters(USER_DAILY_UNITS);+ const service = createYouTubeQuotaService({ quota, now: () => NOW });+ let called = false;+ await assert.rejects(service.run('user-1', 'KEY', async () => { called = true; }), (err: Error) => {+ assert.ok(err instanceof RateLimitedError);+ assert.match(err.message, /midnight Pacific time/);+ return true;+ });+ assert.equal(called, false);+ assert.deepEqual(added, []);+});++test('units spent by a failed operation are still recorded, since YouTube charges failed calls too', async () => {+ const { quota, added } = counters(0);+ const service = createYouTubeQuotaService({ quota, now: () => NOW });+ await assert.rejects(service.run('user-1', 'KEY', async key => { key.spend(1); throw new Error('quotaExceeded'); }), /quotaExceeded/);+ assert.equal(added[0].units, 1);+});++test('without Redis nothing is refused, so analytics keep working', async () => {+ const { quota } = counters(null);+ const service = createYouTubeQuotaService({ quota, now: () => NOW });+ assert.equal(await service.run('user-1', 'KEY', async () => 'ok'), 'ok');+});diff --git a/docs/architecture/backend/CACHING.md b/docs/architecture/backend/CACHING.mdindex 95933b4..dda6877 100644--- a/docs/architecture/backend/CACHING.md+++ b/docs/architecture/backend/CACHING.md@@ -61,6 +61,7 @@ The auth email counters are not cache either. `repositories/emailQuotaRepository|---|---|---|---|| `email:day:{YYYY-MM-DD}` | Emails sent on that UTC day, all users together | Each email the hook sends | 2 days after the last write || `email:to:{sha256 of the lowercased address}:{YYYY-MM-DD}` | Emails sent to one address that day; the key holds a hash, never the address | Each email the hook sends | 2 days after the last write |+| `ytunits:{YYYY-MM-DD}:{userId}` | YouTube Data API units one member spent on that Pacific day (`youtubeQuotaRepository`, on the fail-fast client the rate limits use) | Once at the end of each YouTube operation, with the units of all its calls (search 100, every list call 1), in one pipelined request | 2 days after the last write |One Lua script reads both counters and increments both only when neither cap is reached, so two calls at once cannot both take the last email, and a refused send counts nothing. When Redis is missing or fails, the repository answers `unknown` and the email is sent; Resend's own daily quota is then the cap.diff --git a/docs/architecture/backend/RATE_LIMITING.md b/docs/architecture/backend/RATE_LIMITING.mdindex adc620f..1551858 100644--- a/docs/architecture/backend/RATE_LIMITING.md+++ b/docs/architecture/backend/RATE_LIMITING.md@@ -46,6 +46,8 @@ The limits are sized for the dashboard: an Overview load sends about 20 requestsThe client IP comes from `CF-Connecting-IP`, which Cloudflare always sets and which a client cannot override. Locally, `wrangler dev` does not set it, so the IP rule only applies when a test sends the header. IPv4-mapped IPv6 addresses (`::ffff:203.0.113.7`) are limited as the IPv4 address.+The rules above count requests. The instance's YouTube Data API quota is counted in units instead, per member and Pacific day, by `youtubeQuotaService` (`SERVICES.md`): a member who spent 2,500 units gets 429 with a message that says it resets at midnight Pacific time, so one person can never use up the allowance every other member's analytics depend on.+## Algorithms`backend/lib/rateLimit/algorithms.ts` implements the five standard algorithms as pure functions: given the saved state, the time and the config, each returns the new state and a decision `{ allowed, limit, remaining, resetMs, retryAfterMs }`.diff --git a/docs/architecture/backend/REQUEST_PIPELINE.md b/docs/architecture/backend/REQUEST_PIPELINE.mdindex bb5e669..c93242b 100644--- a/docs/architecture/backend/REQUEST_PIPELINE.md+++ b/docs/architecture/backend/REQUEST_PIPELINE.md@@ -8,7 +8,7 @@ Every API request passes through the same layers, and each layer has one job:| Route table | `backend/routes/` | Maps a method and path to its middleware and controller | HTTP || Controller | `backend/controllers/` | Reads validated input and the user id, calls one service method, picks the status and builds the response | HTTP || Service | `backend/services/` | Business rules and orchestration: what should happen | Application data only |-| Repository | `backend/repositories/` | Data access: one method per query on a Supabase table, the Redis cache or the Redis email counters (`emailQuotaRepository`) | The database |+| Repository | `backend/repositories/` | Data access: one method per query on a Supabase table, the Redis cache, the Redis email counters (`emailQuotaRepository`) or the YouTube quota counters (`youtubeQuotaRepository`) | The database || Request context | `types/context.ts`, `AppVariables` | Request-scoped metadata: request id, user id, cancellation signal, logger | Nothing else |The rule of thumb: HTTP concerns stay in middleware and controllers, business concerns in services, database concerns in repositories, cross-cutting work in middleware and request metadata in the request context.diff --git a/docs/architecture/backend/SERVICES.md b/docs/architecture/backend/SERVICES.mdindex b8ac824..883f030 100644--- a/docs/architecture/backend/SERVICES.md+++ b/docs/architecture/backend/SERVICES.md@@ -50,6 +50,7 @@ A service receives only what it uses: repositories, the cleaned `AppConfig` from| `retention` | `retentionService.ts` | comments, audit | `purge(now?, { commentBatches? })` (daily cron; comments in batches of 1,000) || `mcpAccess` | `mcpAccessService.ts` | MCP access tokens, MCP client grants, audit | `listTokens`, `createToken`, `revokeToken`, `listClients`, `approveClient`, `revokeClient`, `authenticateToken` and `authorizeClient` (used by `requireMcpAuth`) || `mcpFor(principal)` | `mcpService.ts` | (the other services, through the tools in `backend/mcp/`) | `handle(body, headers)`: one JSON-RPC message for one verified caller; lists only the tools the permission allows, asks for confirmation before publishing or deleting, stops a tool after 25 seconds, refuses a result over 40,000 characters, and audits write tools |+| `youtubeQuota` (internal, not on `Services`) | `youtubeQuotaService.ts` | YouTube quota counters (Redis) | `run(userId, apiKey, work)`: refuses a member who spent `USER_DAILY_UNITS` (2,500, a quarter of the default 10,000) on the current Pacific day with `RateLimitedError` (429), hands the work a metered key that `ytFetch` charges per call, and records the units once at the end, even when the work failed. `accounts`, `youtubeAnalytics`, `trackedContent` and `overview` run every YouTube call through it; refresh-all uses one run for all of a member's channels. Without Redis nothing is metered (audit lead L6) || `health` | `healthService.ts` | health | `readiness()` (`GET /api/health/ready`: one database read, timed; returns no data) |Every user-scoped method takes the verified `userId` as its first argument. Methods that need one of the user's records load it with a `...ForUser` repository method and throw `NotFoundError` when it is missing or on the wrong platform, so another user's record is never found. `scheduledPublisher`, `tokenRefresh`, `automationWebhook`, `automationProcessor`, `automationFollowups`, `automationPublicReplies`, `automationReconciler` and `retention` are the only services that use the cross-user repository methods (`listByPlatform`, `findById`, `updateById`, `listDue`, `listStale`, `claim`, and the automation queue, follow-up, review, intake and retention methods).