Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.2 KB
Raw Blame
import { describe, it, expect, beforeEach, afterEach, afterAll, vi } from 'vitest';
import { mkdtemp, rm, writeFile } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';

const TEST_ROOT = await mkdtemp(join(tmpdir(), 'portos-igquota-'));
vi.mock('../lib/fileUtils.js', async () => {
  const actual = await vi.importActual('../lib/fileUtils.js');
  actual.PATHS.data = TEST_ROOT;
  return actual;
});

const {
  parseImageQuotaSignal,
  recordImageGenOutcome,
  getImageGenQuota,
  isQuotaTrackedImageMode,
  initImageGenQuotaHook,
  __resetImageGenQuotaHookForTests,
  __drainImageGenQuotaHookForTests,
} = await import('./imageGenQuota.js');
const { imageGenEvents } = await import('./imageGenEvents.js');
const { checkFabrication } = await import('./imageGen/fabricationGuard.js');

// A real scratch dir with drawing-script residue, so the guard's actual message
// (not a copy of it) is what gets fed to the quota classifier below.
const FAB_DIR = await mkdtemp(join(tmpdir(), 'portos-igquota-fab-'));
await writeFile(join(FAB_DIR, 'render_sheet.py'), 'import matplotlib');

// The verbatim shape Antigravity returned on 2026-07-31 (paths/ids redacted).
// Its "(around 2026-07-31T21:38:09Z)" is an ABSOLUTE instant, and the parser
// deliberately ignores an absolute reset that has already passed
// (`imageGenQuota.js` `parseAbsoluteReset`: `at <= now ? null : at`). So every
// assertion on that instant must pin `now` to before it — otherwise the test
// silently starts exercising the relative-fallback branch instead, and compares
// against the wall clock. Reading it off the real clock is what broke this suite
// on 2026-08-01 (#3265).
const FIXTURE_NOW = Date.parse('2026-07-31T16:38:09Z'); // 5h before the reset above
const AGY_429 ='Agy did not produce an image at the directed path. Agy said: "Here is the error details returned by the API: * **Error Code**: 429 (Resource Exhausted) * **Message**: You have exhausted your capacity on this model. Your quota will reset in approximately 5 hours (around 2026-07-31T21:38:09Z). Since the `generate_image` tool relies entirely on this backend service, I am currently unable to write the requested image to `/tmp/portos-agy-<id>/output.png`. Please try again after the quota resets."';

beforeEach(async () => {
  __resetImageGenQuotaHookForTests();
  imageGenEvents.removeAllListeners();
  await rm(join(TEST_ROOT, 'imagegen-quota.json'), { force: true });
});
afterEach(async () => {
  await rm(join(TEST_ROOT, 'imagegen-quota.json'), { force: true });
});

// TEST_ROOT / FAB_DIR are module-scope roots (top-level-await mkdtemp, before
// any per-test hook runs) — remove the roots themselves once, not just the
// file inside TEST_ROOT the per-test hooks above clean (#9032).
afterAll(async () => {
  await rm(TEST_ROOT, { recursive: true, force: true });
  await rm(FAB_DIR, { recursive: true, force: true });
});

describe('parseImageQuotaSignal', () => {
  it('reads the exact 429 the imagen backend returned', () => {
    const signal = parseImageQuotaSignal(AGY_429, { now: FIXTURE_NOW });
    expect(signal.exhausted).toBe(true);
    // The parenthetical absolute instant beats the "approximately 5 hours".
    expect(new Date(signal.resetsAt).toISOString()).toBe('2026-07-31T21:38:09.000Z');
  });

  it('ignores an absolute reset that has already passed and uses the relative window', () => {
    // The branch that quietly took over when this suite's `now` drifted past the
    // fixture's instant. Pinning it means a future drift fails loudly instead of
    // re-routing another test's assertion into here.
    const now = Date.parse('2026-08-02T00:00:00Z'); // after the message's 21:38:09Z
    const signal = parseImageQuotaSignal(AGY_429, { now });
    expect(signal.exhausted).toBe(true);
    expect(new Date(signal.resetsAt).toISOString()).toBe('2026-08-02T05:00:00.000Z'); // now + 5h
  });

  it('falls back to the relative window when no absolute time is given', () => {
    const now = Date.parse('2026-07-31T16:38:09Z');
    const signal = parseImageQuotaSignal('429: quota will reset in approximately 5 hours.', { now });
    expect(signal.exhausted).toBe(true);
    expect(new Date(signal.resetsAt).toISOString()).toBe('2026-07-31T21:38:09.000Z');
  });

  it('marks a quota block even when no reset time is stated', () => {
    expect(parseImageQuotaSignal('RESOURCE_EXHAUSTED')).toEqual({ exhausted: true, resetsAt: null });
  });

  it('does not treat a content decline as a quota block', () => {
    // Misreading this would show a phantom "0% left" meter with no reset.
    const signal = parseImageQuotaSignal('Agy said: "I cannot generate that image."');
    expect(signal.exhausted).toBe(false);
  });

  it('catches a real rate limit even when the prompt echo contains "credit"', () => {
    // The toolkit's shared classifier is first-match-wins with its billing rule
    // ABOVE its rate-limit rule, so filtering the prompt-echo false positive by
    // category would have suppressed this real refusal. Match phrases directly.
    const s = 'rate limit exceeded for image_gen; prompt was: a wizard holding a credit card';
    expect(parseImageQuotaSignal(s).exhausted).toBe(true);
  });

  it('does not treat a bare number in the prompt as a 429', () => {
    // `\b429\b` unanchored matched artwork descriptions.
    expect(parseImageQuotaSignal('Agy said: "I will not draw route 429 signage."').exhausted).toBe(false);
  });

  it('ignores a past timestamp in the narration', () => {
    // Narration carries log/session stamps. Accepting one as the reset would
    // set blockedUntil to an elapsed instant, which reads as "not blocked".
    const now = Date.parse('2026-07-31T17:00:00Z');
    const s = 'Agy said: "429 Resource Exhausted. [log 2026-07-30T09:00:00Z] retry later."';
    expect(parseImageQuotaSignal(s, { now })).toEqual({ exhausted: true, resetsAt: null });
  });

  it('does not treat prompt content echoed in a decline as a quota block', () => {
    // The classified text is the model's own narration, which quotes the image
    // prompt back. A bare "credit"/"payment" in the artwork description must
    // never read as a billing failure and paint a phantom 0%-left meter.
    const decline = 'Agy said: "I cannot generate that image: a wizard holding a credit card and a payment terminal."';
    expect(parseImageQuotaSignal(decline).exhausted).toBe(false);
  });

  it('still catches a genuine out-of-credits refusal', () => {
    // The precise phrasing survives — only the bare-word match was dropped.
    expect(parseImageQuotaSignal('Render failed: you are out of credits.').exhausted).toBe(true);
  });

  it('catches the rate-limit phrasings the shared CLI classifier also knows', () => {
    expect(parseImageQuotaSignal('Grok said: "too many requests, slow down"').exhausted).toBe(true);
  });

  it('does not treat a crashed CLI as a quota block', () => {
    expect(parseImageQuotaSignal('Agy generation failed: Exit code 1').exhausted).toBe(false);
    expect(parseImageQuotaSignal('Failed to spawn agy: ENOENT').exhausted).toBe(false);
    expect(parseImageQuotaSignal('').exhausted).toBe(false);
  });
});

describe('isQuotaTrackedImageMode', () => {
  it('tracks the cloud CLIs and ignores local/external', () => {
    expect(isQuotaTrackedImageMode('agy')).toBe(true);
    expect(isQuotaTrackedImageMode('grok')).toBe(true);
    expect(isQuotaTrackedImageMode('codex')).toBe(true);
    // Local renders on the user's own GPU — there is no remote quota to report.
    expect(isQuotaTrackedImageMode('local')).toBe(false);
    expect(isQuotaTrackedImageMode('external')).toBe(false);
    // fal.ai is metered per image — no subscription window to report.
    expect(isQuotaTrackedImageMode('fal')).toBe(false);
  });
});

describe('initImageGenQuotaHook', () => {
  // The recorder rides the imageGenEvents bus rather than provider finalizers,
  // so a backend that emits on the bus is tracked without touching its code.
  // An emitter cannot await its listeners, so the assertions below have to wait
  // for the recorder's write by asking the hook when it is done. Sleeping a
  // fixed 20ms instead is what made this suite flake on the Windows runner
  // (#4788): the write is real disk I/O, and `atomicWrite` sleeps between
  // rename retries there when the destination is briefly locked, so it can
  // easily outlast any constant a test picks.
  const flush = () => __drainImageGenQuotaHookForTests();

  // The bus handler stamps its own `Date.now()`, so unlike the `getImageGenQuota`
  // suite below there is no `now` to thread in — freeze the clock instead.
  // `shouldAdvanceTime` keeps the real timers the write path itself may use
  // (`atomicWrite`'s Windows rename-retry sleep) advancing under the fake clock;
  // without it such a write would never settle and the drain would hang.
  beforeEach(() => {
    vi.useFakeTimers({ shouldAdvanceTime: true });
    vi.setSystemTime(FIXTURE_NOW);
  });
  afterEach(() => {
    vi.useRealTimers();
  });

  it('records a refused render announced on the bus', async () => {
    initImageGenQuotaHook();
    imageGenEvents.emit('failed', { mode: 'agy', generationId: 'j1', error: AGY_429 });
    await flush();
    const card = await getImageGenQuota({ enabledModes: ['agy'] });
    expect(card.limits[0].resetsAt).toBe('2026-07-31T21:38:09.000Z');
  });

  it('ignores a local render on the same bus', async () => {
    initImageGenQuotaHook();
    imageGenEvents.emit('completed', { mode: 'local', generationId: 'j2' });
    await flush();
    const card = await getImageGenQuota({ enabledModes: ['agy'] });
    expect(card.metrics[0].value).toBe('No renders · 24h');
  });

  it('subscribes only once across repeated boots', async () => {
    initImageGenQuotaHook();
    initImageGenQuotaHook();
    imageGenEvents.emit('completed', { mode: 'agy', generationId: 'j3' });
    await flush();
    const card = await getImageGenQuota({ enabledModes: ['agy'] });
    // A double subscription would count this single render twice.
    expect(card.metrics[0].value).toBe('1 render · 24h');
  });
});

describe('getImageGenQuota', () => {
  const now = Date.parse('2026-07-31T17:00:00Z');

  it('renders no card at all when no cloud image backend is enabled', async () => {
    // Local renders spend no remote quota — one "should there be a card"
    // decision, made here, rather than a second not-supported state.
    expect(await getImageGenQuota({ enabledModes: ['local'], now })).toBe(null);
    expect(await getImageGenQuota({ enabledModes: [], now })).toBe(null);
  });

  it('marks the card unburnable so quota-burn never targets it', async () => {
    // These cards carry no measurable headroom; a 0%-left image meter must not
    // read to the quota-burn candidate feed as capacity to spend down.
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now });
    expect((await getImageGenQuota({ enabledModes: ['agy'], now })).burnable).toBe(false);
  });

  it('surfaces a real 0%-left meter with the provider-stated reset', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: AGY_429, at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.limits).toHaveLength(1);
    expect(card.limits[0]).toMatchObject({
      label: 'Agy · generate_image',
      percentUsed: 100,
      percentRemaining: 0,
      resetsAt: '2026-07-31T21:38:09.000Z',
    });
  });

  it('reports observed render counts rather than inventing a percentage', async () => {
    // The whole point of the card: a quota we cannot query must never render
    // as a reassuring "100% left" meter.
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now });
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.limits).toEqual([]);
    expect(card.metrics).toHaveLength(1);
    expect(card.metrics[0].value).toBe('2 renders · 24h');
    expect(card.metrics[0].detail).toContain('quota not reported');
  });

  it('shows a block whose refusal stated no reset time', async () => {
    // "blocked, reset unknown" must not collapse into "not blocked" — keying
    // blocked-ness off blockedUntil alone did exactly that.
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: 'RESOURCE_EXHAUSTED', at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.limits).toHaveLength(1);
    expect(card.limits[0]).toMatchObject({ percentRemaining: 0, resetsAt: null });
  });

  it('expires an unknown-reset block instead of sticking forever', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: 'RESOURCE_EXHAUSTED', at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now: now + 2 * 60 * 60 * 1000 });
    expect(card.limits).toEqual([]);
  });

  it('does not let a repeat refusal erase the reset the first one stated', async () => {
    // A retry during an active block returns a bare 429 without restating the
    // reset; overwriting with null would drop the meter mid-block.
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: AGY_429, at: now });
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: '429 RESOURCE_EXHAUSTED, please try later', at: now + 60_000 });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now: now + 120_000 });
    expect(card.limits).toHaveLength(1);
    expect(card.limits[0].resetsAt).toBe('2026-07-31T21:38:09.000Z');
  });

  it('does not let our own fabrication message register as a provider quota block', async () => {
    // The guard's prose is PortOS's inference, not the provider's words. If it
    // self-classifies, every fabrication rejection fakes a quota block — and
    // overwrites a real one.
    const guardMsg = await checkFabrication(FAB_DIR, 'generate_image');
    expect(parseImageQuotaSignal(guardMsg).exhausted).toBe(false);
  });

  it('drops the block once its stated reset has passed', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: AGY_429, at: now });
    const after = Date.parse('2026-07-31T21:40:00Z');
    const card = await getImageGenQuota({ enabledModes: ['agy'], now: after });
    expect(card.limits).toEqual([]);
  });

  it('clears a stale block as soon as a render succeeds again', async () => {
    // Providers round "approximately", so a success is stronger evidence than
    // the stated reset time — without this the card lies until the clock says so.
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: AGY_429, at: now });
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now + 60_000 });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now: now + 120_000 });
    expect(card.limits).toEqual([]);
  });

  it('does not block on a non-quota failure', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: 'Exit code 1', at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.limits).toEqual([]);
    expect(card.metrics[0].detail).toBe('1 failed');
  });

  it('keeps each backend independent', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: AGY_429, at: now });
    await recordImageGenOutcome({ mode: 'grok', ok: true, at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy', 'grok'], now });
    expect(card.limits.map((l) => l.model)).toEqual(['Agy']);
    expect(card.metrics.map((m) => m.label)).toEqual(['Grok · image_gen']);
  });

  it('ignores an untracked mode handed to the recorder', async () => {
    await recordImageGenOutcome({ mode: 'local', ok: false, error: AGY_429, at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.limits).toEqual([]);
  });

  it('survives a process restart', async () => {
    // State lives only on disk — no in-memory mirror to warm.
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: AGY_429, at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.limits[0].resetsAt).toBe('2026-07-31T21:38:09.000Z');
  });

  it('reports an unreadable ledger as an error, not as zero renders', async () => {
    // "Could not read" must never render as a reassuring "No renders · 24h" —
    // that would also silently hide an active block.
    await writeFile(join(TEST_ROOT, 'imagegen-quota.json'), '{ this is not json');
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.error).toContain('Could not read');
    expect(card.metrics).toEqual([]);
  });

  it('ages renders out of the 24h window', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now - 25 * 60 * 60 * 1000 });
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now });
    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    expect(card.metrics[0].value).toBe('1 render · 24h');
  });

  it('reports counts across 24h, week (7d), and month (30d) periods', async () => {
    // 10 days ago (outside 7d, inside 30d)
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now - 10 * 24 * 60 * 60 * 1000 });
    // 3 days ago (outside 24h, inside 7d & 30d)
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now - 3 * 24 * 60 * 60 * 1000 });
    // 2 hours ago (inside 24h, 7d & 30d)
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now - 2 * 60 * 60 * 1000 });

    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    const metric = card.metrics[0];
    expect(metric.value).toBe('1 render · 24h');
    expect(metric.counts).toEqual({ day: 1, week: 2, month: 3 });
    expect(metric.periods['24h'].value).toBe('1 render · 24h');
    expect(metric.periods['7d'].value).toBe('2 renders · week');
    expect(metric.periods['30d'].value).toBe('3 renders · month');
  });

  it('selects week or month as the primary period when requested', async () => {
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now - 3 * 24 * 60 * 60 * 1000 });
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: now - 2 * 60 * 60 * 1000 });

    const weekCard = await getImageGenQuota({ enabledModes: ['agy'], now, period: 'week' });
    expect(weekCard.period).toBe('7d');
    expect(weekCard.metrics[0].value).toBe('2 renders · week');

    const monthCard = await getImageGenQuota({ enabledModes: ['agy'], now, period: '30d' });
    expect(monthCard.period).toBe('30d');
    expect(monthCard.metrics[0].value).toBe('2 renders · month');
  });

  it('tracks renders since the last observed reset', async () => {
    const resetTime = now - 2 * 60 * 60 * 1000;
    // Before reset
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: resetTime - 60_000 });
    // Simulate active block with reset
    await recordImageGenOutcome({ mode: 'agy', ok: false, error: `quota will reset in approximately 2 hours (around ${new Date(resetTime).toISOString()})`, at: resetTime - 7200_000 });
    // Successful render after reset
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: resetTime + 60_000 });

    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    const metric = card.metrics[0];
    expect(metric.periods.reset).toBeDefined();
    expect(metric.periods.reset.count).toBe(1);
    expect(metric.periods.reset.value).toBe('1 render · reset');

    const resetCard = await getImageGenQuota({ enabledModes: ['agy'], now, period: 'reset' });
    expect(resetCard.period).toBe('reset');
    expect(resetCard.metrics[0].value).toBe('1 render · reset');
  });

  it('reconciles reset boundary when rendering recovers early before forecasted reset', async () => {
    const forecastedReset = now + 2 * 60 * 60 * 1000;
    const earlyRecoveryTime = now - 30 * 60 * 1000;
    // Failed render with forecasted reset 2 hours into future
    await recordImageGenOutcome({
      mode: 'agy',
      ok: false,
      error: `quota will reset in approximately 2 hours (around ${new Date(forecastedReset).toISOString()})`,
      at: now - 60 * 60 * 1000,
    });
    // Early successful render before forecastedReset
    await recordImageGenOutcome({ mode: 'agy', ok: true, at: earlyRecoveryTime });

    const card = await getImageGenQuota({ enabledModes: ['agy'], now });
    const metric = card.metrics[0];
    expect(metric.periods.reset).toBeDefined();
    expect(metric.periods.reset.count).toBe(1);
    expect(metric.periods.reset.value).toBe('1 render · reset');
  });
});