Skip to content

A spy satellite simulator in your browser, except the data is real. Live open source spatial intelligence on a photorealistic 3D globe.

23.5 KB
Raw Blame
import test from 'node:test';
import assert from 'node:assert/strict';
import { createVoiceInactivity } from './inactivity.js';
import { RealtimeTurns } from '../realtimeTurns.js';
import { observeNativeActivity } from './nativeActivity.js';
// Deliberately small native-shaped fixtures exercise wrapper contracts, not devices.
function gemini(onChange = () => {}) {
  const callbacks = [];
  const input = {
    pushToTalkKeyHeld: false,
    setMicrophoneEnabled() {},
    releasePushToTalkKey() {
      this.pushToTalkKeyHeld = false;
    },
    resetSession() {
      this.pushToTalkKeyHeld = false;
    },
  };
  const c = {
    input,
    captureOwner: { admitted: false },
    createAudio(opts) {
      callbacks.push(opts);
      return {
        pending: 0,
        play() {
          this.pending++;
          return 42;
        },
        clearPlayback() {
          this.pending = 0;
        },
        close() {
          this.pending = 0;
        },
      };
    },
    handleMessage() {},
  };
  const observer = observeNativeActivity({
    adapter: { controller: c },
    nativeProvider: 'gemini',
    onChange,
  });
  const start = () => {
    observer.handle({ type: 'state', state: 'connecting' });
    c.audio = c.createAudio({ pushToTalk: true });
    observer.handle({ type: 'state', state: 'listening' });
    return c.audio;
  };
  return { c, input, observer, callbacks, start };
}
test('custom host never inspected, including hostile controller getter', () => {
  const adapter = {
    get controller() {
      throw Error('must not inspect');
    },
  };
  const o = observeNativeActivity({ adapter, nativeProvider: null });
  assert.equal(o.snapshot().known, false);
  o.handle({ type: 'state', state: 'listening' });
  o.destroy();
});
test('PTT hold is direct before connect; silent automatic stream is not admitted speech', () => {
  const f = gemini();
  f.input.pushToTalkKeyHeld = true;
  assert.equal(f.observer.snapshot().held, true);
  assert.equal(f.observer.snapshot().connected, false);
  f.start();
  f.callbacks[0].onAudioStart({});
  assert.equal(f.observer.snapshot().inputBusy, false);
  f.c.captureOwner.admitted = true;
  f.callbacks[0].onAudioStart({});
  assert.equal(f.observer.snapshot().inputBusy, true);
  f.callbacks[0].onAudioDiscard();
  f.input.releasePushToTalkKey();
  assert.equal(f.observer.snapshot().inputBusy, false);
  assert.equal(f.observer.snapshot().held, false);
});
test('queue-to-drain, clear and close use current instance, without an acoustic claim', () => {
  const f = gemini();
  const a = f.start();
  assert.equal(a.play(), 42);
  assert.equal(f.observer.snapshot().outputBusy, true);
  a.clearPlayback();
  assert.equal(f.observer.snapshot().outputBusy, false);
  a.play();
  a.close();
  assert.equal(f.observer.snapshot().outputBusy, false);
  a.play();
  a.pending = 0;
  f.callbacks[0].onDrain();
  assert.equal(f.observer.snapshot().outputBusy, false);
});
test('same-mount restart old callbacks cannot clear new admitted capture/output', () => {
  const f = gemini();
  f.start();
  const old = f.callbacks[0];
  const b = f.start();
  f.c.captureOwner.admitted = true;
  f.callbacks[1].onAudioStart();
  b.play();
  old.onAudioDiscard();
  old.onDrain();
  assert.equal(f.observer.snapshot().inputBusy, true);
  assert.equal(f.observer.snapshot().outputBusy, true);
});
test('native callbacks original first with receiver, arguments, return and thrown identity; observation failure isolated', () => {
  const f = gemini(() => {
    throw Error('observer');
  });
  const receiver = {};
  const argument = {};
  let seen;
  f.c.audio = f.c.createAudio({
    pushToTalk: true,
    onAudioEnd: function (...args) {
      seen = [this, args];
      return argument;
    },
  });
  const cb = f.callbacks[0];
  assert.equal(cb.onAudioEnd.call(receiver, argument), argument);
  assert.equal(seen[0], receiver);
  assert.equal(seen[1][0], argument);
  const error = Error('same');
  f.c.audio = f.c.createAudio({
    onDrain() {
      throw error;
    },
  });
  assert.throws(
    () => f.callbacks[1].onDrain(),
    (e) => e === error,
  );
});
test('reentrant Stop in original callback suppresses stale post-callback observation; teardown preserves newer replacement', () => {
  let count = 0;
  const f = gemini(() => count++);
  f.c.audio = f.c.createAudio({
    onDrain() {
      f.observer.handle({ type: 'state', state: 'idle' });
    },
  });
  const before = count;
  f.callbacks[0].onDrain();
  assert.equal(count, before + 1);
  assert.equal(f.observer.snapshot().connected, false);
  const newer = () => {};
  f.input.releasePushToTalkKey = newer;
  f.observer.destroy();
  assert.equal(f.input.releasePushToTalkKey, newer);
});
test('OpenAI response.done does not drain output; absent lifecycle remains unknown', () => {
  const turns = {
    assistantAudioPlaying: false,
    responseActive: false,
    observeSpeech(e) {
      if (e.type === 'output_audio_buffer.started')
        this.assistantAudioPlaying = true;
      if (
        e.type === 'output_audio_buffer.stopped' ||
        e.type === 'output_audio_buffer.cleared'
      )
        this.assistantAudioPlaying = false;
    },
  };
  const c = { _turns: turns, _input: { pushToTalkKeyHeld: false } };
  const original = turns.observeSpeech;
  const o = observeNativeActivity({
    adapter: { controller: c },
    nativeProvider: 'openai',
  });
  o.handle({ type: 'state', state: 'listening' });
  assert.equal(o.snapshot().known, false);
  turns.observeSpeech({ type: 'output_audio_buffer.started' });
  assert.equal(o.snapshot().known, true);
  turns.responseActive = false;
  o.handle({ type: 'completion' });
  assert.equal(o.snapshot().outputBusy, true);
  turns.observeSpeech({ type: 'output_audio_buffer.cleared' });
  assert.equal(o.snapshot().outputBusy, false);
  o.destroy();
  assert.equal(turns.observeSpeech, original);
});
test('method receiver/args/result and throw are exact; abort restores wrappers', () => {
  const abort = new AbortController();
  const result = Promise.resolve();
  let seen;
  const error = Error('original');
  const input = {
    setMicrophoneEnabled(...args) {
      seen = [this, args];
      if (args[0] === 'throw') throw error;
      return result;
    },
  };
  const original = input.setMicrophoneEnabled;
  const c = {
    input,
    createAudio() {
      return {};
    },
    handleMessage() {},
  };
  const o = observeNativeActivity({
    adapter: { controller: c },
    nativeProvider: 'gemini',
    signal: abort.signal,
    onChange() {
      throw Error('observe');
    },
  });
  assert.equal(input.setMicrophoneEnabled(1, 2), result);
  assert.equal(seen[0], input);
  assert.deepEqual(seen[1], [1, 2]);
  assert.throws(
    () => input.setMicrophoneEnabled('throw'),
    (e) => e === error,
  );
  abort.abort();
  assert.equal(input.setMicrophoneEnabled, original);
  assert.equal(o.snapshot().known, false);
});

test('options-method callback receiver maps to original options, explicit receiver is preserved', () => {
  const f = gemini();
  let receiver;
  const options = {
    onDrain() {
      receiver = this;
      return 17;
    },
  };
  f.c.audio = f.c.createAudio(options);
  assert.equal(f.callbacks[0].onDrain(), 17);
  assert.equal(receiver, options);
  const explicit = {};
  f.callbacks[0].onDrain.call(explicit);
  assert.equal(receiver, explicit);
});
test('actual native RealtimeTurns output lifecycle remains busy beyond generation', () => {
  const turns = new RealtimeTurns({});
  const observer = observeNativeActivity({
    adapter: { controller: { _turns: turns, _input: {} } },
    nativeProvider: 'openai',
  });
  observer.handle({ type: 'state', state: 'listening' });
  turns.observeSpeech({ type: 'output_audio_buffer.started' });
  turns.responseActive = false;
  observer.handle({ type: 'completion' });
  assert.equal(observer.snapshot().outputBusy, true);
  turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
  assert.equal(observer.snapshot().outputBusy, false);
  turns.observeSpeech({ type: 'response.created' });
  turns.observeSpeech({ type: 'output_audio_buffer.stopped' });
  assert.equal(observer.snapshot().known, false);
  observer.destroy();
});

test('verified clean native mount can arm initial idle, eventless response invalidates it', () => {
  const turns = new RealtimeTurns({});
  const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
  const observer = observeNativeActivity({
    adapter: { controller },
    nativeProvider: 'openai',
  });
  let armed = 0,
    cleared = 0;
  const idle = createVoiceInactivity({
    readActivity: observer.snapshot,
    readPending: () => 0,
    setTimer() {
      armed++;
      return 1;
    },
    clearTimer() {
      cleared++;
    },
  });
  assert.equal(observer.snapshot().known, true);
  assert.equal(armed, 0);
  controller.status = 'listening';
  controller.dc = { readyState: 'open' };
  observer.handle({ type: 'state', state: 'listening' });
  idle.update();
  assert.equal(armed, 1);
  turns.observeSpeech({ type: 'response.created' });
  idle.update();
  assert.equal(observer.snapshot().known, false);
  assert.equal(cleared, 1);
  turns.responseActive = false;
  observer.handle({ type: 'completion' });
  idle.update();
  assert.equal(armed, 1);
  assert.equal(observer.snapshot().known, false);
  controller.status = 'idle';
  controller.dc = null;
  turns.reset();
  observer.handle({ type: 'state', state: 'idle' });
  assert.equal(observer.snapshot().known, true);
  idle.destroy();
  observer.destroy();
});
test('attaching to already active native transport never assumes empty playback', () => {
  const turns = new RealtimeTurns({});
  for (const controller of [
    { status: 'listening', dc: {}, _turns: turns, _input: {} },
    { status: 'idle', dc: {}, _turns: turns, _input: {} },
  ]) {
    const observer = observeNativeActivity({
      adapter: { controller },
      nativeProvider: 'openai',
    });
    assert.equal(observer.snapshot().known, false);
    observer.destroy();
  }
  turns.responseActive = true;
  const observer = observeNativeActivity({
    adapter: {
      controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
    },
    nativeProvider: 'openai',
  });
  assert.equal(observer.snapshot().known, false);
  observer.destroy();
});

test('Gemini queue-only work blocks idle, queue drain rearms, expiry rereads delayed queue', () => {
  const f = gemini();
  f.start();
  f.c.controlQueue = [];
  let callback,
    arms = 0,
    clears = 0,
    expired = 0;
  const idle = createVoiceInactivity({
    readActivity: f.observer.snapshot,
    readPending: () => 0,
    onExpire() {
      expired++;
    },
    setTimer(fn) {
      callback = fn;
      return ++arms;
    },
    clearTimer() {
      clears++;
    },
  });
  assert.equal(arms, 1);
  f.c.controlQueue.push({});
  assert.equal(f.observer.snapshot().generationBusy, true);
  idle.update();
  assert.equal(clears, 1);
  f.c.controlQueue.shift();
  assert.equal(f.observer.snapshot().generationBusy, false);
  idle.update();
  assert.equal(arms, 2);
  // A queue may change after the last event-driven snapshot and before the deadline.
  f.c.controlQueue.push({});
  callback();
  assert.equal(expired, 0);
  f.c.controlQueue = [];
  idle.update();
  assert.equal(arms, 3);
  callback();
  assert.equal(expired, 1);
  idle.destroy();
  f.observer.destroy();
});

test('Gemini presentation-only pending does not block idle; retired/deferred work still does', () => {
  const f = gemini();
  f.start();
  f.c.userTurnPending = true;
  assert.equal(f.observer.snapshot().generationBusy, false);
  f.c.retiredReplyPending = true;
  assert.equal(f.observer.snapshot().generationBusy, true);
  f.c.retiredReplyPending = false;
  f.c.deferredInput = {};
  assert.equal(f.observer.snapshot().generationBusy, true);
  f.c.deferredInput = null;
  assert.equal(f.observer.snapshot().generationBusy, false);
  f.observer.destroy();
});

test('actual Gemini startup phases keep playback observation and permit quiet open-mic expiry', async () => {
  const { harness } = await import('../../testSupport/geminiNativeHarness.mjs');
  let observer,
    idle,
    deadline,
    expiries = 0;
  const h = harness({ emit: (event) => observer?.handle(event) });
  observer = observeNativeActivity({
    adapter: { controller: h.c },
    nativeProvider: 'gemini',
    onChange: () => idle?.update(),
  });
  idle = createVoiceInactivity({
    readActivity: observer.snapshot,
    readPending: () => 0,
    setTimer(fn) {
      deadline = fn;
      return fn;
    },
    clearTimer() {},
    onExpire() {
      expiries++;
    },
  });
  try {
    await h.c.start();
    assert.equal(h.c.userTurnPending, true);
    assert.equal(idle.snapshot().eligible, true);
    const beforeOutput = deadline;
    h.sockets[0].message({
      serverContent: {
        modelTurn: {
          parts: [
            {
              inlineData: { data: 'output', mimeType: 'audio/pcm;rate=24000' },
            },
          ],
        },
        turnComplete: true,
      },
    });
    assert.equal(observer.snapshot().outputBusy, true);
    assert.equal(idle.snapshot().eligible, false);
    beforeOutput();
    assert.equal(expiries, 0);
    h.captures[0].drain();
    assert.equal(observer.snapshot().outputBusy, false);
    assert.equal(idle.snapshot().eligible, true);
    deadline();
    assert.equal(expiries, 1);
  } finally {
    idle.destroy();
    observer.destroy();
    h.c.stop({ removeUi: true });
  }
});

test('native typed input observation preserves receiver/result, refuses failed submissions and retires old wrapper', () => {
  let inputs = 0;
  const controller = {
    input: {},
    handleMessage() {},
    createAudio() {},
    sendTextCommand(text) {
      assert.equal(this, controller);
      if (text === 'throw') throw new Error('rejected');
      return text === 'reject' ? false : undefined;
    },
  };
  const original = controller.sendTextCommand;
  const observer = observeNativeActivity({
    adapter: { controller },
    nativeProvider: 'gemini',
    onInput: () => inputs++,
  });
  const old = controller.sendTextCommand;
  assert.equal(controller.sendTextCommand('typed request'), undefined);
  controller.sendTextCommand('reject');
  controller.sendTextCommand('');
  assert.throws(() => controller.sendTextCommand('throw'), /rejected/);
  assert.equal(inputs, 1);
  observer.destroy();
  assert.equal(controller.sendTextCommand, original);
  old.call(controller, 'late');
  assert.equal(inputs, 1);
});

test('explicit silent OpenAI completion rearms expiry, including cancelled and failed responses', () => {
  for (const status of ['completed', 'cancelled', 'failed', 'incomplete']) {
    const turns = new RealtimeTurns({});
    const controller = { status: 'idle', dc: null, _turns: turns, _input: {} };
    let idle;
    let timer;
    let expired = 0;
    const observer = observeNativeActivity({
      adapter: { controller },
      nativeProvider: 'openai',
      onChange: () => idle?.update(),
    });
    idle = createVoiceInactivity({
      readActivity: observer.snapshot,
      readPending: () => false,
      setTimer(callback) {
        timer = callback;
        return 1;
      },
      clearTimer() {
        timer = null;
      },
      onExpire: () => expired++,
    });
    controller.status = 'listening';
    controller.dc = {};
    observer.handle({ type: 'state', state: 'listening' });
    assert.equal(typeof timer, 'function');
    turns.responseActive = true;
    turns.observeSpeech({
      type: 'response.created',
      response: { id: 'silent' },
    });
    assert.equal(timer, null);
    turns.observeSpeech({
      type: 'response.done',
      response: {
        id: 'silent',
        status,
        output:
          status === 'completed'
            ? [{ type: 'function_call', name: 'set_panel_open' }]
            : [],
      },
    });
    // The provider updates generation state after forwarding the raw event.
    assert.equal(timer, null);
    turns.responseActive = false;
    observer.handle({ type: 'completion' });
    assert.equal(observer.snapshot().known, true);
    assert.equal(typeof timer, 'function');
    timer();
    assert.equal(expired, 1);
    idle.destroy();
    observer.destroy();
  }
});

test('silent OpenAI replies cannot clear unresolved older audio or incomplete output metadata', () => {
  const turns = new RealtimeTurns({});
  const observer = observeNativeActivity({
    adapter: {
      controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
    },
    nativeProvider: 'openai',
  });
  const created = (id) =>
    turns.observeSpeech({ type: 'response.created', response: { id } });
  const done = (id, output) =>
    turns.observeSpeech({
      type: 'response.done',
      response: { id, status: 'completed', output },
    });
  created('audio');
  turns.observeSpeech({
    type: 'response.output_audio.delta',
    response_id: 'audio',
  });
  created('silent');
  done('silent', []);
  assert.equal(observer.snapshot().known, false);
  done('audio', []); // Even contradictory completion metadata cannot erase audio evidence.
  assert.equal(observer.snapshot().known, false);
  turns.observeSpeech({
    type: 'output_audio_buffer.started',
    response_id: 'audio',
  });
  assert.equal(observer.snapshot().outputBusy, true);
  done('silent', []);
  assert.equal(observer.snapshot().outputBusy, true);
  turns.observeSpeech({
    type: 'output_audio_buffer.stopped',
    response_id: 'audio',
  });
  assert.equal(observer.snapshot().outputBusy, false);
  created('unknown');
  done('unknown', undefined);
  assert.equal(observer.snapshot().known, false);
  done('unobserved', []);
  assert.equal(observer.snapshot().known, false);
  observer.destroy();
});

test('overlapping silent replies resolve independently and audio output items remain unknown until playback', () => {
  const turns = new RealtimeTurns({});
  const observer = observeNativeActivity({
    adapter: {
      controller: { status: 'idle', dc: null, _turns: turns, _input: {} },
    },
    nativeProvider: 'openai',
  });
  for (const id of ['a', 'b'])
    turns.observeSpeech({ type: 'response.created', response: { id } });
  for (const id of ['a', 'b']) {
    turns.observeSpeech({
      type: 'response.done',
      response: {
        id,
        status: 'completed',
        output: [
          { type: 'message', content: [{ type: 'text', text: 'Done' }] },
        ],
      },
    });
    assert.equal(observer.snapshot().known, id === 'b');
  }
  turns.observeSpeech({ type: 'response.created', response: { id: 'audio' } });
  turns.observeSpeech({
    type: 'response.done',
    response: {
      id: 'audio',
      status: 'completed',
      output: [{ type: 'message', content: [{ type: 'audio' }] }],
    },
  });
  assert.equal(observer.snapshot().known, false);
  observer.destroy();
});

test('silent completion cannot establish output knowledge when observer attached mid-session', () => {
  const turns = new RealtimeTurns({});
  const observer = observeNativeActivity({
    adapter: {
      controller: { status: 'listening', dc: {}, _turns: turns, _input: {} },
    },
    nativeProvider: 'openai',
  });
  turns.observeSpeech({ type: 'response.created', response: { id: 'new' } });
  turns.observeSpeech({
    type: 'response.done',
    response: { id: 'new', status: 'completed', output: [] },
  });
  assert.equal(observer.snapshot().known, false);
  observer.destroy();
});

test('Gemini restores each retired audio immediately across repeated same-mount restarts', () => {
  const originals = new Map();
  const controller = {
    input: {},
    createAudio() {
      const audio = { pending: 0, play() {}, clearPlayback() {}, close() {} };
      originals.set(audio, { ...audio });
      return audio;
    },
    handleMessage() {},
  };
  const observer = observeNativeActivity({
    adapter: { controller },
    nativeProvider: 'gemini',
  });
  const assertRestored = (audio) => {
    for (const key of ['play', 'clearPlayback', 'close'])
      assert.equal(audio[key], originals.get(audio)[key]);
  };
  for (let i = 0; i < 5; i++) {
    observer.handle({ type: 'state', state: 'connecting' });
    controller.audio = controller.createAudio({});
    const audio = controller.audio;
    observer.handle({ type: 'state', state: 'listening' });
    assert.notEqual(audio.close, originals.get(audio).close);
    audio.close();
    assertRestored(audio);
    controller.audio = null;
    observer.handle({ type: 'state', state: 'idle' });
  }
  controller.audio = controller.createAudio({});
  const replaced = controller.audio;
  controller.audio = controller.createAudio({});
  assertRestored(replaced);
  const current = controller.audio;
  const replacement = () => {};
  current.play = replacement;
  observer.handle({ type: 'state', state: 'error' });
  assert.equal(current.play, replacement);
  assert.equal(current.close, originals.get(current).close);
  observer.destroy();
});

test('actual OpenAI failed-response error/listening recovery preserves only proven no-audio knowledge', async (t) => {
  const { GevRealtimeController } = await import('../realtimeController.js');
  t.mock.method(console, 'error', () => {});
  for (const [output, known] of [
    [[], true],
    [[{ type: 'message', content: [{ type: 'audio' }] }], false],
    [undefined, false],
  ]) {
    let observer, idle, timer;
    const states = [];
    let expired = 0;
    const controller = new GevRealtimeController({
      runner: async () => ({ ok: true }),
      backend: {},
      debugSink: null,
      ui: {
        root: {
          dataset: {},
          classList: { remove() {} },
          querySelectorAll: () => [],
        },
        status: {},
        detail: {},
      },
      onSessionEvent(event) {
        if (event.type === 'state') states.push(event.state);
        observer?.handle(event);
      },
    });
    observer = observeNativeActivity({
      adapter: { controller },
      nativeProvider: 'openai',
      onChange: () => idle?.update(),
    });
    idle = createVoiceInactivity({
      readActivity: observer.snapshot,
      readPending: () => false,
      setTimer(callback) {
        timer = callback;
        return 1;
      },
      clearTimer() {
        timer = null;
      },
      onExpire() {
        expired++;
      },
    });
    try {
      controller.dc = {
        readyState: 'open',
        send() {},
        close() {
          this.readyState = 'closed';
        },
      };
      controller.setStatus('listening', 'Ready');
      const event = (payload) =>
        controller.handleRealtimeEvent({ data: JSON.stringify(payload) });
      await event({
        type: 'response.created',
        response: { id: 'failed-reply' },
      });
      await event({
        type: 'response.done',
        response: {
          id: 'failed-reply',
          status: 'failed',
          output,
          status_details: { error: { message: 'Synthetic failed reply' } },
        },
      });
      assert.deepEqual(states.slice(-2), ['error', 'listening']);
      assert.equal(controller.dc.readyState, 'open');
      assert.equal(observer.snapshot().known, known);
      assert.equal(idle.snapshot().eligible, known);
      if (known) {
        timer();
        assert.equal(expired, 1);
      } else assert.equal(timer, null);
      // Fatal teardown can preserve the error status; the next connection must
      // still retire the old generation rather than inherit unresolved output.
      controller.setStatus('error', 'Transport failed');
      controller.stop({ preserveStatus: true });
      controller.setStatus('connecting', 'Reconnecting');
      assert.equal(observer.snapshot().connected, false);
      assert.equal(observer.snapshot().known, true);
      controller.stop();
      assert.equal(observer.snapshot().connected, false);
      assert.equal(observer.snapshot().known, true);
    } finally {
      idle.destroy();
      observer.destroy();
      controller.stop();
    }
  }
});