Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。
TypeScript agentai-agentai-assistantandroidclaude-codecodexelectroniosllmmacosreact-nativetypescriptwindows
import { describe, expect, it } from 'vitest';
import {
holdStandaloneStopTokenDelta,
isPossibleStandaloneStopPrefix,
stableInternalWebCitationBoundary,
stableStandaloneModelStopTokenBoundary,
stripInternalWebCitations,
stripStandaloneModelStopToken,
} from './internalCitation.js';
const source1 = '\uE200cite\uE202turn17search1\uE201';
const source2 = '\uE200cite\uE202turn17search1\uE202turn17search2\uE201';
describe('internal Web citation normalization', () => {
it('strips single and multiple-source markers without changing punctuation', () => {
expect(stripInternalWebCitations(`结论。${source1}`)).toBe('结论。');
expect(stripInternalWebCitations(`A ${source1};B ${source2}。`)).toBe('A ;B 。');
});
it('leaves ordinary cite text and unrelated private-use text untouched', () => {
expect(stripInternalWebCitations('Please cite the source.')).toBe('Please cite the source.');
expect(stripInternalWebCitations('ordinary \uE200 text')).toBe('ordinary \uE200 text');
});
it('is idempotent and strips unfinished final tails', () => {
expect(stripInternalWebCitations(`done ${source1}`)).toBe('done ');
expect(stripInternalWebCitations(stripInternalWebCitations(`done ${source1}`))).toBe('done ');
expect(stripInternalWebCitations('done \uE200cite\uE202turn17sea')).toBe('done ');
expect(stripInternalWebCitations('done \uE200ci')).toBe('done ');
});
it('holds split streaming prefixes and incomplete markers until they are complete', () => {
expect(stableInternalWebCitationBoundary('done')).toBe(4);
expect(stableInternalWebCitationBoundary('done \uE200ci')).toBe(5);
expect(stableInternalWebCitationBoundary('done \uE200cite\uE202turn17sea')).toBe(5);
expect(stableInternalWebCitationBoundary(`done ${source2}`)).toBe(`done ${source2}`.length);
});
});
describe('leaked model stop tokens', () => {
it('strips a whole-message Grok stop token to empty text', () => {
expect(stripStandaloneModelStopToken('<|eos|>')).toBe('');
expect(stripInternalWebCitations(' <|eos|>\n')).toBe('');
});
it('strips a whole-message run of stop tokens, including mixed tokens and blank lines', () => {
expect(stripStandaloneModelStopToken('<|eos|><|eos|>')).toBe('');
expect(stripInternalWebCitations('<|eos|>\n\n<|eos|>')).toBe('');
expect(stripInternalWebCitations(' <|eos|> <|eot_id|>\n')).toBe('');
expect(stripInternalWebCitations('<|eos|>answer')).toBe('<|eos|>answer');
});
it('leaves completed messages that start with a stop token but continue as prose', () => {
expect(stripStandaloneModelStopToken('<|eos|>answer')).toBe('<|eos|>answer');
expect(stripInternalWebCitations('<|endoftext|> The token is still here')).toBe(
'<|endoftext|> The token is still here',
);
});
it('leaves embedded stop-token prose intact', () => {
expect(stripInternalWebCitations('The token is <|eos|>')).toBe('The token is <|eos|>');
expect(stripInternalWebCitations('done<|eot_id|>')).toBe('done<|eot_id|>');
expect(stripInternalWebCitations('use <|placeholder|> here')).toBe('use <|placeholder|> here');
});
it('holds an incomplete standalone prefix until the closer arrives', () => {
expect(stableStandaloneModelStopTokenBoundary('hello')).toBe(5);
expect(stableStandaloneModelStopTokenBoundary('<|eo')).toBe(0);
expect(stableStandaloneModelStopTokenBoundary(' <|eo')).toBe(0);
expect(stableStandaloneModelStopTokenBoundary('<|eos|>')).toBe(0);
expect(stableStandaloneModelStopTokenBoundary('<|eos|><|eos|>')).toBe(0);
expect(stableStandaloneModelStopTokenBoundary('<|eos|><|eo')).toBe(0);
expect(stableStandaloneModelStopTokenBoundary('The token is <|eo')).toBe('The token is <|eo'.length);
expect(isPossibleStandaloneStopPrefix(' <|eo')).toBe(true);
expect(isPossibleStandaloneStopPrefix('<')).toBe(true);
expect(isPossibleStandaloneStopPrefix('<|')).toBe(true);
expect(isPossibleStandaloneStopPrefix(`${' '.repeat(23)}<|eos|>`)).toBe(true);
expect(isPossibleStandaloneStopPrefix('<|eos|><|eos|>')).toBe(true);
expect(isPossibleStandaloneStopPrefix('The token is <|eo')).toBe(false);
});
it('is idempotent', () => {
expect(stripInternalWebCitations(stripInternalWebCitations('<|eos|>'))).toBe('');
expect(stripInternalWebCitations(stripInternalWebCitations('<|eos|><|eos|>'))).toBe('');
expect(stripInternalWebCitations(stripInternalWebCitations('The token is <|eos|>'))).toBe(
'The token is <|eos|>',
);
});
it('holds a split leftover on one stream without mixing another stream', () => {
const a = { pending: '', emitted: false };
const b = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(a, '<|eo')).toBeNull();
expect(holdStandaloneStopTokenDelta(b, 'answer')).toBe('answer');
expect(holdStandaloneStopTokenDelta(a, 's|>')).toBeNull();
expect(a).toEqual({ pending: '<|eos|>', emitted: false });
expect(b).toEqual({ pending: '', emitted: true });
});
it('holds a long-padded leftover token until the closer arrives', () => {
const buffer = { pending: '', emitted: false };
const padded = `${' '.repeat(23)}<|eos|>`;
expect(holdStandaloneStopTokenDelta(buffer, padded)).toBeNull();
expect(buffer.emitted).toBe(false);
expect(buffer.pending.endsWith('<|eos|>')).toBe(true);
});
it('holds whitespace-only deltas without releasing a later leftover token', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, ' '.repeat(23))).toBeNull();
expect(buffer.pending).toBe(' '.repeat(23));
expect(holdStandaloneStopTokenDelta(buffer, '<|eos|>')).toBeNull();
expect(buffer.emitted).toBe(false);
expect(buffer.pending.trim()).toBe('<|eos|>');
});
it('keeps ordinary leading whitespace when later prose arrives', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, ' ')).toBeNull();
expect(holdStandaloneStopTokenDelta(buffer, 'code')).toBe(' code');
});
it('holds a one-character leftover prefix until the closer arrives', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, '<')).toBeNull();
expect(holdStandaloneStopTokenDelta(buffer, '|eos|>')).toBeNull();
expect(buffer).toEqual({ pending: '<|eos|>', emitted: false });
});
it('keeps a split leftover so later prose reconstructs the original text', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, '<|eos|>')).toBeNull();
expect(holdStandaloneStopTokenDelta(buffer, 'answer')).toBe('<|eos|>answer');
});
it('drops a completed leftover so a second copy cannot flush it as prose', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, '<|eos|>')).toBeNull();
expect(holdStandaloneStopTokenDelta(buffer, '<|eos|>')).toBeNull();
expect(buffer.emitted).toBe(false);
expect(buffer.pending.trim()).toBe('<|eos|><|eos|>');
expect(holdStandaloneStopTokenDelta(buffer, 'answer')).toBe('<|eos|><|eos|>answer');
});
it('still strips a split web citation after ordinary prose has been emitted', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, '结论。')).toBe('结论。');
expect(holdStandaloneStopTokenDelta(buffer, '\uE200ci')).toBeNull();
expect(holdStandaloneStopTokenDelta(buffer, 'te\uE202turn17search1\uE201 后续')).toBe(' 后续');
expect(buffer.emitted).toBe(true);
expect(buffer.pending).toBe('');
});
it('keeps an embedded stop token after ordinary prose has been emitted', () => {
const buffer = { pending: '', emitted: false };
expect(holdStandaloneStopTokenDelta(buffer, 'The token is ')).toBe('The token is ');
expect(holdStandaloneStopTokenDelta(buffer, '<|eos|>')).toBe('<|eos|>');
});
});