Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
3.8 KB
Raw Blame
/**
 * LoCoMo → LongMemEval conversion (src/eval/longmemeval/locomo.ts) on a tiny
 * synthetic conversation.
 *
 * Protects: one question per non-adversarial QA; evidence dialog ids map to
 * the stable per-conversation session ids; the haystack, its dates and the
 * question date come from the conversation's sessions; speaker names and
 * photo captions survive into the turns; an empty or sealed conversation list
 * is refused unless the custodian asks; QAs without usable evidence are
 * skipped and counted.
 */
import { describe, expect, test } from 'bun:test';
import { convertLocomo, locomoDate, type LocomoSample } from '../src/eval/longmemeval/locomo.ts';
import { haystackToPages, normalizeSessions } from '../src/eval/longmemeval/adapter.ts';

const sample: LocomoSample = {
  sample_id: 'conv-90',
  conversation: {
    speaker_a: 'Alice', speaker_b: 'Bob',
    session_1_date_time: '1:10 pm on 27 March, 2023',
    session_1: [
      { speaker: 'Alice', dia_id: 'D1:1', text: 'I adopted a puppy.' },
      { speaker: 'Bob', dia_id: 'D1:2', text: 'Look at this!', blip_caption: 'a photo of a red bike' },
    ],
    session_2_date_time: '9:02 am on 2 April, 2023',
    session_2: [{ speaker: 'Bob', dia_id: 'D2:1', text: 'I started a new job.' }],
  },
  qa: [
    { question: 'What did Alice adopt?', answer: 'a puppy', evidence: ['D1:1'], category: 4 },
    { question: 'Which bird does Alice like?', evidence: ['D1:1'], category: 5 },
    { question: 'What happened in March and April?', answer: 2023, evidence: ['D2:1', 'D1:2', 'D1:1'], category: 1 },
    { question: 'Unsupported?', answer: 'x', evidence: ['D9:1'], category: 2 },
  ],
};

describe('convertLocomo', () => {
  test('one question per non-adversarial QA, gold sessions from evidence ids', () => {
    const { questions, skipped } = convertLocomo([sample], ['conv-90']);
    expect(questions.map(q => q.question_id)).toEqual(['conv-90_q0', 'conv-90_q2']);
    expect(skipped['conv-90']).toEqual({ adversarial: 1, no_evidence: 1 });
    const [q0, q2] = questions;
    expect(q0.question_type).toBe('single-hop');
    expect(q0.answer_session_ids).toEqual(['conv-90_s1']);
    expect(q2.question_type).toBe('multi-hop');
    expect(q2.answer).toBe('2023');
    expect(q2.answer_session_ids).toEqual(['conv-90_s1', 'conv-90_s2']);
    expect(q0.haystack_session_ids).toEqual(['conv-90_s1', 'conv-90_s2']);
    expect(q0.haystack_dates).toEqual(['2023/03/27 (Mon) 13:10', '2023/04/02 (Sun) 09:02']);
    expect(q0.question_date).toBe('2023/04/02 (Sun) 09:02');
    expect(q0.haystack_sessions[0]).toEqual([
      { role: 'Alice', content: 'I adopted a puppy.' },
      { role: 'Bob', content: 'Look at this! [shares a photo of a red bike]' },
    ]);
  });

  test('the harness adapter reads the converted shape', () => {
    const [q] = convertLocomo([sample], ['conv-90']).questions;
    expect(normalizeSessions(q as never).map(s => s.session_id)).toEqual(['conv-90_s1', 'conv-90_s2']);
    expect(haystackToPages(q as never)[0].content).toContain('**Alice:** I adopted a puppy.');
  });

  test('an empty, unknown or sealed conversation list is refused', () => {
    expect(() => convertLocomo([sample], [])).toThrow('explicit conversation list');
    expect(() => convertLocomo([sample], ['conv-91'])).toThrow('not in the input');
    expect(() => convertLocomo([{ ...sample, sample_id: 'conv-26' }], ['conv-26'])).toThrow('sealed LoCoMo split');
    expect(convertLocomo([{ ...sample, sample_id: 'conv-26' }], ['conv-26'], { custodian: true }).questions).toHaveLength(2);
  });

  test('locomoDate reads the LoCoMo date format and rejects others', () => {
    expect(locomoDate('12:05 am on 1 January, 2024')).toBe('2024/01/01 (Mon) 00:05');
    expect(locomoDate('12:30 pm on 15 June, 2023')).toBe('2023/06/15 (Thu) 12:30');
    expect(() => locomoDate('yesterday')).toThrow('unrecognized');
  });
});