Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
fix(eval): anchor the nightly probe fixture relative-date questions with a question_date (#7308, #7501)
garrytand5d4a511f1d0
diff --git a/test/fixtures/longmemeval-nightly.jsonl b/test/fixtures/longmemeval-nightly.jsonlindex 0a04281b3f..9532e78681 100644--- a/test/fixtures/longmemeval-nightly.jsonl+++ b/test/fixtures/longmemeval-nightly.jsonl@@ -1,10 +1,10 @@-{"question_id":"nightly-1","question_type":"single-session-user","question":"Where did alice-example say she lived last year?","answer":"in widget-co","haystack_sessions":[[{"role":"user","content":"alice-example mentioned she lived in widget-co last year"},{"role":"assistant","content":"Got it, noted alice-example lived in widget-co"}]],"haystack_session_ids":["nightly-1-s1"],"haystack_dates":["2025-08-15"],"answer_session_ids":["nightly-1-s1"]}-{"question_id":"nightly-2","question_type":"multi-session","question":"What project was charlie-example working on last month?","answer":"the component-library-example","haystack_sessions":[[{"role":"user","content":"charlie-example started a new project called component-library-example"}],[{"role":"user","content":"charlie-example shipped the component-library-example v1"}]],"haystack_session_ids":["nightly-2-s1","nightly-2-s2"],"haystack_dates":["2025-07-01","2025-07-15"],"answer_session_ids":["nightly-2-s1","nightly-2-s2"]}-{"question_id":"nightly-3","question_type":"temporal-reasoning","question":"When did bob-example join fund-a-example?","answer":"2024-06-01","haystack_sessions":[[{"role":"user","content":"bob-example joined fund-a-example on 2024-06-01 as a partner"}]],"haystack_session_ids":["nightly-3-s1"],"haystack_dates":["2024-06-01"],"answer_session_ids":["nightly-3-s1"]}-{"question_id":"nightly-4","question_type":"knowledge-update","question":"What is dana-example's current role?","answer":"CTO at acme-example","haystack_sessions":[[{"role":"user","content":"dana-example was previously CEO at widget-co"}],[{"role":"user","content":"dana-example took the CTO role at acme-example last week"}]],"haystack_session_ids":["nightly-4-s1","nightly-4-s2"],"haystack_dates":["2024-01-15","2025-03-20"],"answer_session_ids":["nightly-4-s2"]}-{"question_id":"nightly-5","question_type":"single-session-assistant","question":"What city was the demo-day-example held in?","answer":"placeholder-city","haystack_sessions":[[{"role":"user","content":"I attended demo-day-example yesterday"},{"role":"assistant","content":"You mentioned demo-day-example was held in placeholder-city this year"}]],"haystack_session_ids":["nightly-5-s1"],"haystack_dates":["2025-05-01"],"answer_session_ids":["nightly-5-s1"]}-{"question_id":"nightly-6","question_type":"single-session-user","question":"What language is elliot-example most fluent in?","answer":"TypeScript","haystack_sessions":[[{"role":"user","content":"elliot-example is most fluent in TypeScript; uses it for the bun-runtime-example"}]],"haystack_session_ids":["nightly-6-s1"],"haystack_dates":["2025-04-10"],"answer_session_ids":["nightly-6-s1"]}-{"question_id":"nightly-7","question_type":"multi-session","question":"How many projects has alice-example launched this year?","answer":"three","haystack_sessions":[[{"role":"user","content":"alice-example launched widget-co/payments in January"}],[{"role":"user","content":"alice-example launched widget-co/identity in March"}],[{"role":"user","content":"alice-example launched widget-co/reporting in June"}]],"haystack_session_ids":["nightly-7-s1","nightly-7-s2","nightly-7-s3"],"haystack_dates":["2025-01-15","2025-03-20","2025-06-12"],"answer_session_ids":["nightly-7-s1","nightly-7-s2","nightly-7-s3"]}-{"question_id":"nightly-8","question_type":"temporal-reasoning","question":"Was the rag-example paper read before or after the architecture-example notes?","answer":"before","haystack_sessions":[[{"role":"user","content":"Read the rag-example paper today"}],[{"role":"user","content":"Wrote architecture-example notes a week later"}]],"haystack_session_ids":["nightly-8-s1","nightly-8-s2"],"haystack_dates":["2025-02-01","2025-02-08"],"answer_session_ids":["nightly-8-s1","nightly-8-s2"]}-{"question_id":"nightly-9","question_type":"knowledge-update","question":"What is charlie-example's preferred testing framework now?","answer":"bun:test","haystack_sessions":[[{"role":"user","content":"charlie-example used vitest for years"}],[{"role":"user","content":"charlie-example switched to bun:test as primary testing framework"}]],"haystack_session_ids":["nightly-9-s1","nightly-9-s2"],"haystack_dates":["2023-09-01","2025-03-05"],"answer_session_ids":["nightly-9-s2"]}-{"question_id":"nightly-10","question_type":"single-session-assistant","question":"What is the demo-day-example pitch length?","answer":"three minutes","haystack_sessions":[[{"role":"user","content":"How long are demo-day-example pitches?"},{"role":"assistant","content":"demo-day-example pitches are three minutes per founder."}]],"haystack_session_ids":["nightly-10-s1"],"haystack_dates":["2025-04-22"],"answer_session_ids":["nightly-10-s1"]}+{"question_id":"nightly-1","question_type":"single-session-user","question":"Where did alice-example say she lived last year?","answer":"in widget-co","haystack_sessions":[[{"role":"user","content":"alice-example mentioned she lived in widget-co last year"},{"role":"assistant","content":"Got it, noted alice-example lived in widget-co"}]],"haystack_session_ids":["nightly-1-s1"],"haystack_dates":["2025-08-15"],"question_date":"2025-08-20","answer_session_ids":["nightly-1-s1"]}+{"question_id":"nightly-2","question_type":"multi-session","question":"What project was charlie-example working on last month?","answer":"the component-library-example","haystack_sessions":[[{"role":"user","content":"charlie-example started a new project called component-library-example"}],[{"role":"user","content":"charlie-example shipped the component-library-example v1"}]],"haystack_session_ids":["nightly-2-s1","nightly-2-s2"],"haystack_dates":["2025-07-01","2025-07-15"],"question_date":"2025-08-20","answer_session_ids":["nightly-2-s1","nightly-2-s2"]}+{"question_id":"nightly-3","question_type":"temporal-reasoning","question":"When did bob-example join fund-a-example?","answer":"2024-06-01","haystack_sessions":[[{"role":"user","content":"bob-example joined fund-a-example on 2024-06-01 as a partner"}]],"haystack_session_ids":["nightly-3-s1"],"haystack_dates":["2024-06-01"],"question_date":"2025-08-20","answer_session_ids":["nightly-3-s1"]}+{"question_id":"nightly-4","question_type":"knowledge-update","question":"What is dana-example's current role?","answer":"CTO at acme-example","haystack_sessions":[[{"role":"user","content":"dana-example was previously CEO at widget-co"}],[{"role":"user","content":"dana-example took the CTO role at acme-example last week"}]],"haystack_session_ids":["nightly-4-s1","nightly-4-s2"],"haystack_dates":["2024-01-15","2025-03-20"],"question_date":"2025-08-20","answer_session_ids":["nightly-4-s2"]}+{"question_id":"nightly-5","question_type":"single-session-assistant","question":"What city was the demo-day-example held in?","answer":"placeholder-city","haystack_sessions":[[{"role":"user","content":"I attended demo-day-example yesterday"},{"role":"assistant","content":"You mentioned demo-day-example was held in placeholder-city this year"}]],"haystack_session_ids":["nightly-5-s1"],"haystack_dates":["2025-05-01"],"question_date":"2025-08-20","answer_session_ids":["nightly-5-s1"]}+{"question_id":"nightly-6","question_type":"single-session-user","question":"What language is elliot-example most fluent in?","answer":"TypeScript","haystack_sessions":[[{"role":"user","content":"elliot-example is most fluent in TypeScript; uses it for the bun-runtime-example"}]],"haystack_session_ids":["nightly-6-s1"],"haystack_dates":["2025-04-10"],"question_date":"2025-08-20","answer_session_ids":["nightly-6-s1"]}+{"question_id":"nightly-7","question_type":"multi-session","question":"How many projects has alice-example launched this year?","answer":"three","haystack_sessions":[[{"role":"user","content":"alice-example launched widget-co/payments in January"}],[{"role":"user","content":"alice-example launched widget-co/identity in March"}],[{"role":"user","content":"alice-example launched widget-co/reporting in June"}]],"haystack_session_ids":["nightly-7-s1","nightly-7-s2","nightly-7-s3"],"haystack_dates":["2025-01-15","2025-03-20","2025-06-12"],"question_date":"2025-08-20","answer_session_ids":["nightly-7-s1","nightly-7-s2","nightly-7-s3"]}+{"question_id":"nightly-8","question_type":"temporal-reasoning","question":"Was the rag-example paper read before or after the architecture-example notes?","answer":"before","haystack_sessions":[[{"role":"user","content":"Read the rag-example paper today"}],[{"role":"user","content":"Wrote architecture-example notes a week later"}]],"haystack_session_ids":["nightly-8-s1","nightly-8-s2"],"haystack_dates":["2025-02-01","2025-02-08"],"question_date":"2025-08-20","answer_session_ids":["nightly-8-s1","nightly-8-s2"]}+{"question_id":"nightly-9","question_type":"knowledge-update","question":"What is charlie-example's preferred testing framework now?","answer":"bun:test","haystack_sessions":[[{"role":"user","content":"charlie-example used vitest for years"}],[{"role":"user","content":"charlie-example switched to bun:test as primary testing framework"}]],"haystack_session_ids":["nightly-9-s1","nightly-9-s2"],"haystack_dates":["2023-09-01","2025-03-05"],"question_date":"2025-08-20","answer_session_ids":["nightly-9-s2"]}+{"question_id":"nightly-10","question_type":"single-session-assistant","question":"What is the demo-day-example pitch length?","answer":"three minutes","haystack_sessions":[[{"role":"user","content":"How long are demo-day-example pitches?"},{"role":"assistant","content":"demo-day-example pitches are three minutes per founder."}]],"haystack_session_ids":["nightly-10-s1"],"haystack_dates":["2025-04-22"],"question_date":"2025-08-20","answer_session_ids":["nightly-10-s1"]}diff --git a/test/longmemeval-nightly-fixture-dates.test.ts b/test/longmemeval-nightly-fixture-dates.test.tsnew file mode 100644index 0000000000..6710261c70--- /dev/null+++ b/test/longmemeval-nightly-fixture-dates.test.ts@@ -0,0 +1,35 @@+/**+ * #7308 / #7501: the nightly quality probe's fixture asks relative-date+ * questions ("last year", "this year", "last month") about a haystack dated+ * 2025. Without a `question_date` the reader has no `Current Date:` line and+ * resolves them against the real clock, so the probe failed every night on+ * the calendar, not the brain. Every row with a relative-date question must+ * carry a question_date on or after its last haystack session, and the+ * reader must render it.+ */+import { expect, test } from 'bun:test';+import { NIGHTLY_PROBE_FIXTURES } from '../src/core/cycle/nightly-probe-fixtures.ts';+import { loadDataset } from '../src/eval/longmemeval/run-config.ts';+import { generateAnswer } from '../src/eval/longmemeval/reader.ts';+import { datasetDay } from '../src/eval/longmemeval/retrieval-arms.ts';+import type { ThinkLLMClient } from '../src/core/think/index.ts';++const RELATIVE_DATE = /\b(last|this|next|past)\s+(year|month|week|quarter)\b|\b(yesterday|today|tomorrow|ago|recently|currently|now)\b/i;++test('#7308: every nightly fixture row with a relative-date question carries a question_date the reader renders', async () => {+ const { questions } = loadDataset(NIGHTLY_PROBE_FIXTURES.longMemEval, 'unused');+ const relative = questions.filter(q => RELATIVE_DATE.test(q.question));+ expect(relative.map(q => q.question_id)).toEqual(expect.arrayContaining(['nightly-1', 'nightly-2', 'nightly-7']));+ for (const q of relative) {+ const day = datasetDay(q.question_date);+ expect({ id: q.question_id, day }).toEqual({ id: q.question_id, day: expect.stringMatching(/^\d{4}-\d{2}-\d{2}$/) });+ for (const haystackDay of (q.haystack_dates ?? []).map(d => datasetDay(d)!)) expect(day! >= haystackDay).toBe(true);+ let sent = '';+ const client = { create: async (request: { messages: Array<{ content: string }> }) => {+ sent = request.messages[0]!.content;+ return { content: [{ type: 'text', text: 'ok' }], stop_reason: 'end_turn', model: 'stub', usage: { input_tokens: 1, output_tokens: 1 } };+ } } as unknown as ThinkLLMClient;+ await generateAnswer(client, q, [], [], new Map(), 'stub');+ expect(sent).toContain(`Current Date: ${q.question_date}\n`);+ }+});