Skip to content

A spy satellite simulator in your browser, except the data is real. Live open source spatial intelligence on a photorealistic 3D globe.

9.4 KB
Raw Blame
/** Inspected native capabilities only; scheduled/reported output is not acoustics. */
export function observeNativeActivity({
  adapter,
  nativeProvider,
  signal,
  onChange,
  onInput,
}) {
  const native = nativeProvider === 'openai' || nativeProvider === 'gemini';
  // Do not even inspect custom controller properties.
  const controller = native ? adapter.controller : null;
  const input = native
    ? nativeProvider === 'gemini'
      ? controller.input
      : controller._input
    : null;
  let dead = false,
    connected = false,
    inputBusy = false,
    recoverableError = false;
  function cleanNativeOutput() {
    if (nativeProvider === 'gemini') return true;
    if (nativeProvider !== 'openai') return false;
    const turns = controller._turns;
    return (
      ['idle', 'connecting', 'error'].includes(controller.status) &&
      controller.dc == null &&
      !turns.assistantAudioPlaying &&
      !turns.responseActive &&
      !turns.responseCreatePending &&
      !turns.userTurnPending &&
      !turns.activeResponseId &&
      !turns.progressResponse
    );
  }
  let outputKnown = cleanNativeOutput();
  let audio = null,
    version = 0;
  const restores = [];
  const audioRestores = [];
  const pendingOutput = new Map();
  let untrackedOutput = !outputKnown;
  function retireAudio() {
    audio = null;
    for (const restore of audioRestores.splice(0).reverse()) restore();
  }
  function refreshOutputKnowledge() {
    outputKnown = !untrackedOutput && pendingOutput.size === 0;
  }
  function hasNoAudioOutput(response) {
    return (
      ['completed', 'cancelled', 'failed', 'incomplete'].includes(
        response?.status,
      ) &&
      Array.isArray(response.output) &&
      response.output.every(
        (item) =>
          item?.type === 'function_call' ||
          (item?.type === 'message' &&
            Array.isArray(item.content) &&
            item.content.every((part) =>
              ['text', 'output_text', 'refusal'].includes(part?.type),
            )),
      )
    );
  }
  function snapshot() {
    const turns = nativeProvider === 'openai' ? controller?._turns : controller;
    return {
      known: native && outputKnown && !dead,
      connected: connected && !dead,
      held: Boolean(input?.pushToTalkKeyHeld),
      inputBusy: Boolean(
        inputBusy ||
        (nativeProvider === 'gemini' && controller?.userActivityOpen),
      ),
      outputBusy: Boolean(
        nativeProvider === 'gemini'
          ? audio && audio === controller.audio && audio.pending
          : turns?.assistantAudioPlaying,
      ),
      generationBusy: Boolean(
        turns?.responseActive ||
        turns?.responseCreatePending ||
        (nativeProvider === 'openai' && turns?.userTurnPending) ||
        (nativeProvider === 'gemini' &&
          (turns?.replyPending ||
            turns?.retiredReplyPending ||
            turns?.deferredInput ||
            turns?.automaticTurnOpen ||
            turns?.pendingTools ||
            turns?.controlQueue?.length)),
      ),
    };
  }
  function notify() {
    if (!dead) {
      try {
        onChange?.(snapshot());
      } catch {}
    }
  }
  function wrap(object, key, after, ownedRestores = restores) {
    const original = object?.[key];
    if (typeof original !== 'function') return;
    function wrapped(...args) {
      const stamp = version;
      const result = Reflect.apply(original, this, args);
      if (!dead && stamp === version) {
        try {
          after?.(args, result);
        } catch {}
      }
      return result;
    }
    object[key] = wrapped;
    ownedRestores.push(() => {
      if (object[key] === wrapped) object[key] = original;
    });
  }
  function handle(event) {
    if (dead || !native) return;
    if (event?.type === 'state') {
      const state = event.state;
      const wasConnected = connected;
      const liveError =
        state === 'error' &&
        nativeProvider === 'openai' &&
        controller.dc?.readyState === 'open';
      const restartingAfterError = state === 'connecting' && recoverableError;
      recoverableError = liveError;
      connected = !['idle', 'connecting', 'error'].includes(state);
      // Connecting has multiple native phases around audio allocation. Retire
      // a previous connected generation once, not each phase of this start.
      // A failed reply can briefly report error on a live OpenAI transport.
      // Preserve its output evidence through recovery to listening; a later
      // reconnect still retires the failed connection's generation.
      if (
        state === 'idle' ||
        (state === 'error' && !liveError) ||
        (state === 'connecting' && (wasConnected || restartingAfterError))
      ) {
        version++;
        retireAudio();
        inputBusy = false;
        outputKnown = cleanNativeOutput();
        untrackedOutput = !outputKnown;
        pendingOutput.clear();
      }
    }
    notify();
  }
  if (native) {
    wrap(controller, 'sendTextCommand', ([text], result) => {
      if (String(text || '').trim() && result !== false) onInput?.();
    });
    for (const key of [
      'setMicrophoneEnabled',
      'releasePushToTalkKey',
      'resetSession',
    ])
      wrap(input, key, notify);
    if (nativeProvider === 'openai') {
      wrap(controller._turns, 'observeSpeech', ([event]) => {
        if (event?.type === 'input_audio_buffer.speech_started')
          inputBusy = true;
        if (event?.type === 'input_audio_buffer.speech_stopped')
          inputBusy = false;
        const type = event?.type;
        const id = event?.response_id || event?.response?.id;
        // Keep each unresolved response separate: a later silent reply cannot
        // establish that an earlier audio reply has drained.
        if (type === 'response.created') {
          if (id && pendingOutput.size < 64)
            pendingOutput.set(id, { audioSeen: false });
          else untrackedOutput = true;
        }
        const pending = pendingOutput.get(id);
        if (
          pending &&
          (type?.startsWith('response.output_audio') ||
            type?.startsWith('response.audio') ||
            ['audio', 'output_audio'].includes(event?.part?.type))
        )
          pending.audioSeen = true;
        if (
          type === 'response.done' &&
          pending &&
          !pending.audioSeen &&
          hasNoAudioOutput(event.response)
        )
          pendingOutput.delete(id);
        if (
          type === 'output_audio_buffer.started' ||
          type === 'output_audio_buffer.cleared'
        ) {
          // Native playback events establish the transport's output lifecycle.
          // Generation completion alone never establishes audio drainage.
          untrackedOutput = false;
          if (id) pendingOutput.delete(id);
          else pendingOutput.clear();
        }
        refreshOutputKnowledge();
        notify();
      });
    } else {
      wrap(controller, 'handleMessage', notify);
      const original = controller.createAudio;
      function createAudio(...args) {
        const stamp = version;
        const options = args[0];
        let current;
        const decorated = { ...options };
        for (const key of [
          'onAudioStart',
          'onAudioEnd',
          'onAudioDiscard',
          'onDrain',
        ]) {
          const callback = options?.[key];
          decorated[key] = function (...values) {
            const result =
              typeof callback === 'function'
                ? Reflect.apply(
                    callback,
                    this === decorated ? options : this,
                    values,
                  )
                : undefined;
            if (
              !dead &&
              stamp === version &&
              audio === current &&
              controller.audio === current
            ) {
              if (
                key === 'onAudioStart' &&
                options.pushToTalk &&
                controller.captureOwner?.admitted
              )
                inputBusy = true;
              if (key === 'onAudioEnd' || key === 'onAudioDiscard')
                inputBusy = false;
              notify();
            }
            return result;
          };
        }
        const forwarded = args.slice();
        forwarded[0] = decorated;
        current = Reflect.apply(original, this, forwarded);
        if (!dead && stamp === version) {
          retireAudio();
          audio = current;
          inputBusy = false;
          for (const key of ['play', 'clearPlayback', 'close'])
            wrap(
              current,
              key,
              () => {
                if (audio !== current) return;
                if (key === 'close') {
                  retireAudio();
                  inputBusy = false;
                }
                if (controller.audio === current) notify();
              },
              audioRestores,
            );
        }
        return current;
      }
      controller.createAudio = createAudio;
      restores.push(() => {
        if (controller.createAudio === createAudio)
          controller.createAudio = original;
      });
    }
  }
  function destroy() {
    if (dead) return;
    dead = true;
    version++;
    retireAudio();
    pendingOutput.clear();
    for (const restore of restores.splice(0).reverse()) restore();
    signal?.removeEventListener('abort', destroy);
  }
  signal?.addEventListener('abort', destroy, { once: true });
  if (signal?.aborted) destroy();
  return { snapshot, handle, destroy };
}