Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

6.9 KB
Raw Blame
/**
 * OpenAI Web Search may encode source references in assistant text with
 * private-use delimiters such as `\uE200cite\uE202...\uE201`. Those tokens are
 * transport metadata, not user-facing prose. Some Codex app-server versions
 * expose only the flattened text and omit the structured URL annotations, so
 * client boundaries must remove the opaque marker deterministically.
 *
 * Grok / xAI may also leak stop tokens such as `<|eos|>` as visible
 * assistant text, typically as a wrap-up after a tool-only turn. One token,
 * several in a row, or the same tokens split across stream deltas are still
 * transport control, not user-facing prose. Mentions that follow real
 * prose, such as `The token is <|eos|>`, stay intact.
 */
const MODEL_STOP_TOKEN_LIST = ['<|endoftext|>', '<|eot_id|>', '<|im_end|>', '<|eos|>'];
const LONGEST_MODEL_STOP_TOKEN = Math.max(
  ...MODEL_STOP_TOKEN_LIST.map((token) => token.length),
);
const WEB_CITATION_OPEN = '\uE200cite\uE202';
const WEB_CITATION_CLOSE = '\uE201';

function unfinishedWebCitationOpen(text: string): number {
  let from = 0;
  for (;;) {
    const open = text.indexOf(WEB_CITATION_OPEN, from);
    if (open === -1) return -1;
    const close = text.indexOf(WEB_CITATION_CLOSE, open + WEB_CITATION_OPEN.length);
    if (close === -1) return open;
    from = close + WEB_CITATION_CLOSE.length;
  }
}

function partialWebCitationPrefixStart(text: string): number {
  const maxProbe = Math.min(text.length, WEB_CITATION_OPEN.length - 1);
  for (let length = maxProbe; length > 0; length -= 1) {
    if (text.endsWith(WEB_CITATION_OPEN.slice(0, length))) return text.length - length;
  }
  return text.length;
}

/**
 * Return the append-only prefix of a streaming assistant snapshot. A Web
 * citation tail is withheld until its closing delimiter arrives, because the
 * whole marker will disappear from the visible text once complete.
 */
export function stableInternalWebCitationBoundary(text: string): number {
  const unfinished = unfinishedWebCitationOpen(text);
  return unfinished === -1 ? partialWebCitationPrefixStart(text) : unfinished;
}

/**
 * Strip complete Web citation markers plus an unfinished final marker/prefix.
 * The payload is intentionally treated as opaque: private-use delimiters are
 * the exact compatibility boundary, while ordinary words such as "cite" are
 * left untouched. The transform is idempotent.
 */
export function stripInternalWebCitations(text: string): string {
  return stripStandaloneModelStopToken(stripInternalWebCitationMarkers(text));
}

function stripInternalWebCitationMarkers(text: string): string {
  if (!text.includes('\uE200')) return text;

  const stableEnd = stableInternalWebCitationBoundary(text);
  const stable = stableEnd === text.length ? text : text.slice(0, stableEnd);
  if (!stable.includes(WEB_CITATION_OPEN)) return stable;

  let output = '';
  let from = 0;
  for (;;) {
    const open = stable.indexOf(WEB_CITATION_OPEN, from);
    if (open === -1) return output + stable.slice(from);
    const close = stable.indexOf(WEB_CITATION_CLOSE, open + WEB_CITATION_OPEN.length);
    if (close === -1) return output + stable.slice(from, open);
    output += stable.slice(from, open);
    from = close + WEB_CITATION_CLOSE.length;
  }
}

function matchStopTokenAt(text: string, index: number): string | null {
  for (const token of MODEL_STOP_TOKEN_LIST) {
    if (text.startsWith(token, index)) return token;
  }
  return null;
}

function isIncompleteStopTokenPrefix(text: string): boolean {
  return text.length > 0
    && text.length <= LONGEST_MODEL_STOP_TOKEN
    && MODEL_STOP_TOKEN_LIST.some((token) => token.startsWith(text));
}

/**
 * Eat a leading run of known stop tokens and the whitespace around them.
 * Completed messages only disappear when nothing remains. The streaming
 * hold keeps a recognized run pending until later prose proves the block
 * is not stop-control-only, then emits that original text.
 */
function consumeLeadingStopControl(text: string): { remainder: string; consumedToken: boolean } {
  let index = 0;
  let consumedToken = false;
  while (index < text.length) {
    if ((text[index] ?? '').trim() === '') {
      index += 1;
      continue;
    }
    const token = matchStopTokenAt(text, index);
    if (!token) break;
    consumedToken = true;
    index += token.length;
  }
  return { remainder: text.slice(index), consumedToken };
}

/**
 * Drop a leaked stop-token run only when it is the whole assistant
 * message (optional surrounding whitespace). Repeated or mixed tokens
 * still count. Mentions such as `The token is <|eos|>`, or a token
 * glued to later prose (`<|eos|>answer`), stay intact.
 */
export function stripStandaloneModelStopToken(text: string): string {
  const { remainder, consumedToken } = consumeLeadingStopControl(text);
  return consumedToken && remainder.length === 0 ? '' : text;
}

/**
 * True while `text` is still only stop-token control: whitespace, one or
 * more known tokens, and at most one incomplete prefix (`<`, `<|eo`, …).
 * Used to hold streaming deltas until the leftover can be dropped or flushed.
 */
export function isPossibleStandaloneStopPrefix(text: string): boolean {
  const { remainder } = consumeLeadingStopControl(text);
  return remainder.length === 0 || isIncompleteStopTokenPrefix(remainder);
}

/**
 * Return the append-only prefix of a streaming snapshot. A standalone
 * leftover — including repeated tokens and an incomplete `<|eos` prefix —
 * is withheld until the closer arrives, because the completed control
 * will disappear.
 */
export function stableStandaloneModelStopTokenBoundary(text: string): number {
  return isPossibleStandaloneStopPrefix(text) ? 0 : text.length;
}

export interface StandaloneStopTokenHold {
  pending: string;
  emitted: boolean;
}

/**
 * Hold leftover stop-token control until this stream has either completed
 * it or proven it is ordinary prose. A recognized run stays pending so a
 * later `answer` can still reconstruct `<|eos|>answer`; only a later copy
 * of the same control run is dropped. After visible prose starts, still
 * hold an unfinished Web-citation tail so split markers are not emitted
 * raw. The buffer is per stream; callers must not share it across
 * concurrent text streams.
 */
export function holdStandaloneStopTokenDelta(
  buffer: StandaloneStopTokenHold,
  delta: string,
): string | null {
  const combined = buffer.pending + delta;
  if (!buffer.emitted) {
    const { remainder } = consumeLeadingStopControl(combined);
    if (remainder.length === 0 || isIncompleteStopTokenPrefix(remainder)) {
      buffer.pending = combined;
      return null;
    }
  }
  const citationEnd = stableInternalWebCitationBoundary(combined);
  buffer.pending = combined.slice(citationEnd);
  const stable = combined.slice(0, citationEnd);
  const visible = buffer.emitted
    ? stripInternalWebCitationMarkers(stable)
    : stripInternalWebCitations(stable);
  if (visible.length > 0) buffer.emitted = true;
  return visible.length > 0 ? visible : null;
}