Skip to content

A spy satellite simulator in your browser, except the data is real. Live open source spatial intelligence on a photorealistic 3D globe.

20.9 KB
Raw Blame
import test from 'node:test';
import assert from 'node:assert/strict';
import {
  harness,
  tick,
  deferred,
} from '../testSupport/geminiNativeHarness.mjs';
import { withToolCatalog } from './gevRealtime.js';

const audioContent = (data) => ({
  modelTurn: {
    parts: [{ inlineData: { data, mimeType: 'audio/pcm;rate=24000' } }],
  },
});
const request = (ws, id, name = 'get_current_view_state', args = {}) =>
  ws.message({ toolCall: { functionCalls: [{ id, name, args }] } });
const responses = (ws) =>
  ws.sent.flatMap((m) => m.toolResponse?.functionResponses || []);
const onset = (ws) => {
  ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
  ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
};
const caption = (ws, text) =>
  ws.message({ serverContent: { inputTranscription: { text } } });

for (const [id, name, args] of [
  ['H01', 'get_entity_context', {}],
  ['H02', 'control_radio', { action: 'status' }],
]) {
  test(`${id} native: cancelling a current ${name} query preserves unrelated Radio preflight`, async () => {
    const ready = deferred(),
      query = deferred(),
      executions = [];
    const h = harness({
      runAction: (name, args, options) => {
        executions.push({ name, args, options });
        return args.action === 'play'
          ? Promise.resolve({
              ok: true,
              action: 'control_radio',
              radioAction: 'play',
              radioPlaybackRequested: true,
            })
          : query.promise;
      },
    });
    h.c.radio.radioLayer.playForVoice = () => ready.promise;
    try {
      await h.c.start();
      h.c.sendTextCommand('Play radio');
      const ws = h.sockets[0];
      request(ws, 'owner', 'control_radio', { action: 'play' });
      await h.c.toolQueue;
      ws.message({
        serverContent: { ...audioContent('confirmation'), turnComplete: true },
      });
      h.captures[0].drain();
      assert.equal(h.c.radio.radioHandoffInFlight, true);
      request(ws, 'query', name, args);
      await tick();
      assert.equal(
        executions.length,
        2,
        'current native query may start after provider completion',
      );
      const epoch = h.c.radio.radioHandoffEpoch;
      const pending = h.c.radio.pendingRadioPlaybackResult;
      const attempt = h.c.radio.radioHandoffAttemptId;
      const ownerCurrent = executions[0].options.isCurrent();
      ws.message({ toolCallCancellation: { ids: ['unknown', 'query'] } });
      await h.c.toolQueue;
      assert.equal(executions[1].options.signal.aborted, true);
      assert.equal(executions[1].options.isCurrent(), false);
      assert.equal(h.c.radio.radioHandoffEpoch, epoch);
      assert.equal(executions[0].options.signal.aborted, false);
      assert.equal(executions[0].options.isCurrent(), ownerCurrent);
      assert.equal(h.c.radio.pendingRadioPlaybackResult, pending);
      assert.equal(h.c.radio.radioHandoffAttemptId, attempt);
      assert.equal(h.c.radio.radioHandoffInFlight, true);
      assert.equal(h.state.stopped, 0);
      assert.equal(h.c.pendingTools, 0);
      assert.deepEqual(
        responses(ws).map((r) => r.id),
        ['owner'],
      );
      assert.equal(responses(ws)[0].response.ok, true);
      query.resolve({ ok: true });
      ready.resolve(true);
      await tick();
      assert.equal(h.c.status, 'idle');
      assert.equal(h.state.ducked, false);
      assert.deepEqual(
        responses(ws).map((r) => r.id),
        ['owner'],
      );
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

for (const [id, prior] of [
  ['H03', 'playing'],
  ['H04', 'before-first-output'],
]) {
  test(`${id} native: PTT fences ${prior} reply and recovers the new call after its boundary`, async () => {
    const h = harness();
    try {
      await h.c.start({ pushToTalk: true });
      const ws = h.sockets[0],
        audio = h.captures[0];
      if (prior === 'playing')
        ws.message({ serverContent: audioContent('old') });
      else {
        h.c.input.setMicrophoneEnabled(true);
        audio.callbacks.onAudio(
          { data: 'AAAB', mimeType: 'audio/pcm;rate=16000' },
          audio.metadata,
        );
        h.c.input.setMicrophoneEnabled(false);
      }
      h.c.input.pushToTalkKeyHeld = true;
      assert.equal(h.c.input.bargeIn(), true);
      h.c.input.setMicrophoneEnabled(true);
      h.c.input.setMicrophoneEnabled(false);
      const turn = h.c.turn,
        played = audio.played.length,
        firstEvent = h.events.length;
      ws.message({
        serverContent: {
          ...audioContent('late-old'),
          outputTranscription: { text: 'retired output' },
          inputTranscription: { text: 'new request' },
        },
      });
      request(ws, 'retired-call', 'fly_to_location', { query: 'old target' });
      await h.c.toolQueue;
      assert.equal(audio.played.length, played);
      assert.equal(audio.pending, false);
      assert.equal(h.calls.length, 0);
      assert.equal(
        h.events.slice(firstEvent).some((e) => e.role === 'assistant'),
        false,
      );
      assert.equal(
        h.events
          .slice(firstEvent)
          .some((e) => e.role === 'user' && e.text === 'new request'),
        true,
      );
      ws.message({ serverContent: { interrupted: true } });
      assert.equal(h.c.turn, turn);
      ws.message({ serverContent: audioContent('after-interrupted') });
      assert.equal(
        audio.played.length,
        played,
        'interrupted alone cannot release the fence',
      );
      ws.message({
        serverContent: {
          ...audioContent('boundary-old'),
          outputTranscription: { text: 'still retired' },
          turnComplete: true,
        },
      });
      assert.equal(
        audio.played.length,
        played,
        'coalesced old boundary output is discarded',
      );
      assert.equal(
        h.events.slice(firstEvent).some((e) => e.type === 'completion'),
        false,
      );
      caption(ws, 'unrelated advisory words');
      ws.message({
        toolCall: {
          functionCalls: [
            {
              id: 'retired-call',
              name: 'fly_to_location',
              args: { query: 'replayed old target' },
            },
            {
              id: 'new-call',
              name: 'fly_to_location',
              args: { query: 'new target' },
            },
          ],
        },
      });
      await h.c.toolQueue;
      assert.deepEqual(
        h.calls.map((c) => c.args.query),
        ['new target'],
      );
      assert.equal(h.calls[0].options.userTurn, undefined);
      ws.message({
        serverContent: {
          ...audioContent('new'),
          outputTranscription: { text: 'new reply' },
          turnComplete: true,
        },
      });
      assert.equal(audio.played.at(-1).data, 'new');
      audio.drain();
      assert.equal(
        h.events.findLast((e) => e.type === 'completion').status,
        'completed',
      );
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

test('H05 native: local typed presentation survives audio drain and new input revokes old call ownership', async () => {
  const h = harness();
  try {
    await h.c.start();
    const ws = h.sockets[0],
      audio = h.captures[0];
    h.c.sendTextCommand('Show the current view');
    const response = h.c.currentTypedInput();
    assert.equal(response.text, 'Show the current view');
    assert.equal(
      h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
        .text,
      response.text,
    );
    request(ws, 'first');
    await h.c.toolQueue;
    assert.equal(h.calls[0].options.userTurn, undefined);
    assert.equal(h.calls[0].options.isCurrent(), true);
    ws.message({
      serverContent: {
        ...audioContent('confirmation'),
        outputTranscription: { text: 'Here is the current view' },
        turnComplete: true,
      },
    });
    assert.equal(
      h.events.some((e) => e.type === 'completion'),
      false,
    );
    assert.equal(
      h.events.findLast((e) => e.role === 'assistant').responseId,
      response.responseId,
    );
    audio.drain();
    const completion = h.events.findLast((e) => e.type === 'completion');
    assert.equal(completion.responseId, response.responseId);
    assert.equal(completion.spoken, true);
    assert.equal(completion.audioDrained, true);
    assert.equal(h.c.currentTypedInput(), response);
    assert.equal(response.text, 'Show the current view');
    request(ws, 'same-turn');
    await h.c.toolQueue;
    assert.equal(h.calls[1].options.userTurn, undefined);
    assert.equal(h.c.currentTypedInput(), response);
    h.c.sendTextCommand('New request');
    assert.notEqual(h.c.currentTypedInput(), response);
    assert.equal(h.c.currentTypedInput().text, 'New request');
    assert.equal(h.calls[0].options.isCurrent(), false);
    assert.equal(h.calls[1].options.isCurrent(), false);
  } finally {
    h.c.stop();
  }
  assert.equal(h.timers.size, 0);
});

test('H08 native: replacing pending runner work cannot underflow tool counts or block successor', async () => {
  const gate = deferred(),
    executions = [];
  const h = harness({
    runAction: (name, args, options) => {
      executions.push({ name, args, options });
      return options.callId === 'old'
        ? gate.promise
        : Promise.resolve({ ok: true });
    },
  });
  try {
    await h.c.start();
    const ws = h.sockets[0];
    onset(ws);
    request(ws, 'old');
    await tick();
    const oldWork = h.c.toolQueue;
    assert.equal(h.c.pendingTools, 1);
    ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
    assert.equal(h.c.pendingTools, 0);
    await oldWork;
    assert.equal(h.c.pendingTools, 0);
    assert.equal(executions[0].options.signal.aborted, true);
    assert.equal(executions[0].options.isCurrent(), false);
    ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
    ws.message({ serverContent: { interrupted: true, turnComplete: true } });
    request(ws, 'fresh');
    await h.c.toolQueue;
    assert.equal(executions.at(-1).options.callId, 'fresh');
    assert.equal(h.c.pendingTools, 0);
    gate.resolve({ ok: true });
    await tick();
    assert.equal(h.c.pendingTools, 0);
    assert.deepEqual(
      responses(ws).map((r) => r.id),
      ['fresh'],
    );
    ws.message({
      serverContent: { ...audioContent('fresh reply'), turnComplete: true },
    });
    h.captures[0].drain();
    assert.equal(
      h.events.findLast((e) => e.type === 'completion').status,
      'completed',
    );
  } finally {
    h.c.stop();
  }
  assert.equal(h.timers.size, 0);
});

test('H10 native: held speech dispatches without caption authority and new text revokes its call', async () => {
  const gate = deferred(),
    executions = [];
  const h = harness({
    runAction: (name, args, options) => {
      executions.push({ name, args, options });
      return gate.promise;
    },
  });
  try {
    await h.c.start({ pushToTalk: true });
    const ws = h.sockets[0],
      audio = h.captures[0];
    h.c.input.bargeIn();
    h.c.input.setMicrophoneEnabled(true);
    audio.callbacks.onAudio(
      { data: 'AA==', mimeType: 'audio/pcm;rate=16000' },
      audio.metadata,
    );
    h.c.input.setMicrophoneEnabled(false);
    caption(ws, 'Show the current view');
    assert.equal(h.c.currentTypedInput(), null);
    assert.equal(
      h.events.findLast((e) => e.type === 'transcript' && e.role === 'user')
        .text,
      'Show the current view',
    );
    request(ws, 'held');
    await tick();
    const oldWork = h.c.toolQueue;
    assert.equal(executions.length, 1);
    assert.equal(executions[0].options.userTurn, undefined);
    assert.equal(executions[0].options.isCurrent(), true);
    h.c.sendTextCommand('new intent');
    await oldWork;
    assert.equal(executions[0].options.signal.aborted, true);
    assert.equal(executions[0].options.isCurrent(), false);
    gate.resolve({ ok: true });
    await tick();
    assert.equal(
      responses(ws).some((r) => r.id === 'held'),
      false,
    );
  } finally {
    h.c.stop();
  }
  assert.equal(h.timers.size, 0);
});

for (const [id, restart] of [
  ['H11', false],
  ['H12', true],
]) {
  test(`${id} native: pending tool cannot revive voice after Stop${restart ? '/restart' : ''}`, async () => {
    const gate = deferred(),
      mutations = [],
      executions = [];
    const h = harness({
      runAction: async (name, args, options) => {
        executions.push(options);
        await gate.promise;
        if (options.isCurrent()) mutations.push(name);
        return { ok: true };
      },
    });
    try {
      await h.c.start();
      const ws = h.sockets[0];
      onset(ws);
      request(ws, 'old');
      await tick();
      const pending = h.c.toolQueue;
      assert.equal(h.c.pendingTools, 1);
      h.c.stop();
      const boundary = h.events.length;
      const successor = restart ? h.c.start() : null;
      await pending;
      assert.equal(
        h.events
          .slice(boundary)
          .some((e) => e.type === 'state' && e.state === 'listening'),
        false,
      );
      assert.equal(executions[0].signal.aborted, true);
      assert.equal(executions[0].isCurrent(), false);
      assert.equal(h.c.pendingTools, 0);
      if (successor) await successor;
      const laterBoundary = h.events.length;
      gate.resolve();
      caption(ws, 'late old speech');
      await tick();
      assert.deepEqual(mutations, []);
      assert.equal(
        h.events.slice(laterBoundary).some((e) => e.type === 'state'),
        false,
      );
      assert.equal(responses(ws).length, 0);
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

for (const [id, boundary] of [
  ['H14', 'interrupted'],
  ['H15', 'natural'],
]) {
  test(`${id} native: shared runner zoom executes promptly after ${boundary} retired boundary`, async () => {
    const mutations = [],
      observed = [];
    const guarded = withToolCatalog(
      async (name, args) => {
        mutations.push({ name, args });
        return { ok: true };
      },
      async () => null,
    );
    const h = harness({
      runAction: (name, args, options) => {
        observed.push(options);
        return guarded(name, args, options);
      },
    });
    try {
      await h.c.start();
      const ws = h.sockets[0];
      ws.message({ serverContent: audioContent('old reply') });
      ws.message({ voiceActivity: { type: 'ACTIVITY_START' } });
      ws.message({
        serverContent: {
          ...(boundary === 'interrupted' ? { interrupted: true } : {}),
          turnComplete: true,
        },
      });
      ws.message({ voiceActivity: { type: 'ACTIVITY_END' } });
      caption(ws, 'zoom in');
      request(ws, 'zoom', 'adjust_camera_zoom', { direction: 'in' });
      await tick();
      assert.equal(
        mutations.length,
        1,
        'native action executes without a transcript wait',
      );
      assert.equal(observed[0].userTurn, undefined);
      assert.equal(h.c.pendingTools, 0);
      assert.equal(
        responses(ws).some((r) => r.response.voiceIntentUnverified),
        false,
      );
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

for (const [id, exit] of [
  ['H18', 'stop'],
  ['H19', 'replacement'],
]) {
  test(`${id} native: ${exit} prevents queued camera dispatch and late captions cannot revive it`, async () => {
    const mutations = [],
      observed = [];
    const guarded = withToolCatalog(
      async (name) => {
        mutations.push(name);
        return { ok: true };
      },
      async () => null,
    );
    const h = harness({
      runAction: (name, args, options) => {
        observed.push(name);
        return guarded(name, args, options);
      },
    });
    try {
      await h.c.start();
      const ws = h.sockets[0];
      onset(ws);
      request(ws, 'queued-old', 'adjust_camera_zoom', { direction: 'in' });
      const oldWork = h.c.toolQueue;
      assert.equal(h.c.pendingTools, 1);
      if (exit === 'stop') h.c.stop();
      else h.c.sendTextCommand('Show the whole globe');
      caption(ws, 'zoom in');
      await oldWork;
      assert.deepEqual(mutations, []);
      assert.deepEqual(observed, []);
      assert.equal(h.c.pendingTools, 0);
      if (exit === 'replacement') {
        ws.message({
          serverContent: { interrupted: true, turnComplete: true },
        });
        request(ws, 'fresh-camera', 'zoom_to_globe');
        await h.c.toolQueue;
        assert.deepEqual(mutations, ['zoom_to_globe']);
        assert.deepEqual(observed, ['zoom_to_globe']);
        assert.equal(responses(ws).at(-1).response.ok, true);
      }
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

for (const settlement of ['queued', 'running']) {
  test(`native: cancelled ${settlement} tool settles a completed provider turn`, async () => {
    const result = deferred();
    let executions = 0;
    const h = harness({
      runAction: () => {
        executions++;
        return result.promise;
      },
    });
    try {
      await h.c.start();
      h.c.sendTextCommand('Read current view');
      const ws = h.sockets[0];
      request(ws, 'cancelled-query');
      if (settlement === 'running') await tick();
      ws.message({ toolCallCancellation: { ids: ['cancelled-query'] } });
      ws.message({ serverContent: { turnComplete: true } });
      await h.c.toolQueue;
      assert.equal(h.c.pendingTools, 0);
      assert.equal(executions, settlement === 'queued' ? 0 : 1);
      assert.deepEqual(responses(ws), []);
      assert.equal(
        h.events.filter((event) => event.type === 'completion').length,
        1,
      );
      assert.equal(h.c.pendingCompletionStatus, null);
      result.resolve({ ok: true });
      await tick();
      h.captures[0].drain();
      assert.equal(
        h.events.filter((event) => event.type === 'completion').length,
        1,
      );
      assert.deepEqual(responses(ws), []);
    } finally {
      result.resolve({ ok: true });
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

for (const order of ['before', 'after']) {
  test(`native: cancelling queued output ${order} turnComplete emits one completion`, async () => {
    const h = harness();
    try {
      await h.c.start();
      h.c.sendTextCommand('Read current view');
      const ws = h.sockets[0];
      ws.bufferedAmount = 1e9;
      request(ws, 'queued-output');
      await h.c.toolQueue;
      assert.equal(h.c.pendingTools, 0);
      assert.equal(h.c.controlQueue.length, 1);
      assert.equal(responses(ws).length, 0);
      if (order === 'after')
        ws.message({ serverContent: { turnComplete: true } });
      ws.message({ toolCallCancellation: { ids: ['queued-output'] } });
      if (order === 'before')
        ws.message({ serverContent: { turnComplete: true } });
      assert.equal(h.c.controlQueue.length, 0);
      assert.equal(h.c.controlTimer, null);
      assert.equal(h.c.pendingCompletionStatus, null);
      assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
      ws.bufferedAmount = 0;
      h.advance(1000);
      h.captures[0].drain();
      assert.deepEqual(responses(ws), []);
      assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}

for (const successor of ['input', 'stop']) {
  test(`native: queued-output cancellation completion fences reentrant ${successor}`, async () => {
    let h;
    let successorTurn;
    h = harness({
      emit: (event) => {
        h.events.push(event);
        if (event.type !== 'completion') return;
        h.sockets[0].bufferedAmount = 0;
        if (successor === 'stop') h.c.stop();
        else {
          h.c.sendTextCommand('Show the whole globe');
          successorTurn = h.c.turn;
        }
      },
    });
    try {
      await h.c.start();
      h.c.sendTextCommand('Read current view');
      const ws = h.sockets[0];
      ws.bufferedAmount = 1e9;
      request(ws, 'queued-output');
      await h.c.toolQueue;
      ws.message({ serverContent: { turnComplete: true } });
      const callCount = h.calls.length;
      ws.message({
        toolCallCancellation: { ids: ['queued-output'] },
        serverContent: {
          outputTranscription: { text: 'Retired packet caption' },
          turnComplete: true,
        },
        toolCall: {
          functionCalls: [
            { id: 'retired-tool', name: 'get_current_view_state', args: {} },
          ],
        },
      });
      await h.c.toolQueue;
      assert.equal(h.events.filter((e) => e.type === 'completion').length, 1);
      assert.equal(h.calls.length, callCount);
      assert.deepEqual(responses(ws), []);
      assert.ok(
        !h.events.some((e) =>
          JSON.stringify(e).includes('Retired packet caption'),
        ),
      );
      if (successor === 'input') {
        assert.equal(h.c.turn, successorTurn);
        assert.equal(h.c.modelComplete, false);
        assert.equal(h.c.replyPending, true);
      } else assert.equal(h.c.isActive(), false);
    } finally {
      h.c.stop();
    }
    assert.equal(h.timers.size, 0);
  });
}