Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { describe, it, expect, vi, beforeEach } from 'vitest';
// Mock providers before importing the module
vi.mock('./providers.js', () => ({
getActiveProvider: vi.fn(),
getProviderById: vi.fn()
}));
// Mock the central LLM handler — meatspacePostLlm used to spawn child_process
// + fetch directly, but now delegates to runPromptThroughProvider. Runner
// internals (spawn args, --model flag injection) are covered by runner.test.js.
vi.mock('./promptRunner.js', () => ({
assertProvider: (provider, { message, code, status = 503 } = {}) => {
if (provider) return;
const err = new Error(message || 'No AI provider available');
if (code) { err.status = status; err.code = code; }
throw err;
},
runPromptThroughProvider: vi.fn()
}));
import { getActiveProvider, getProviderById } from './providers.js';
import { runPromptThroughProvider } from './promptRunner.js';
import {
LLM_DRILL_TYPES,
generateLlmDrill,
generateWordAssociation,
generateStoryRecall,
generateVerbalFluency,
generateWitComeback,
generatePunWordplay,
generateCompoundChain,
generateBridgeWord,
generateDoubleMeaning,
generateIdiomTwist,
generateWhatIf,
generateAlternativeUses,
generateStoryPrompt,
generateInventionPitch,
generateReframe,
scoreLlmDrill,
callAI,
} from './meatspacePostLlm.js';
// Helper: mock an API provider that returns a given JSON string. Sets the
// central handler to resolve with the stringified response — drills then
// parse it the same way they used to with a fetch-mocked API response.
function mockApiProvider(responseJson) {
const provider = {
id: 'test-provider',
enabled: true,
type: 'api',
endpoint: 'http://localhost:9999',
apiKey: 'test-key',
defaultModel: 'test-model'
};
getActiveProvider.mockResolvedValue(provider);
getProviderById.mockResolvedValue(provider);
runPromptThroughProvider.mockResolvedValue({
text: JSON.stringify(responseJson),
runId: 'test-run',
model: 'test-model'
});
return provider;
}
const storyQuestions = () => [
{ question: 'Who visited?', answer: 'Jane', aliases: ['J. Doe'] },
{ question: 'Where did she go?', answer: 'Paris' },
{ question: 'When did she go?', answer: 'Monday' },
];
const compoundExamples = (root = 'fire') => [
`${root}house`, `${root}work`, `${root}place`, `${root}wood`, `${root}fly`,
`${root}proof`, `${root}storm`, `${root}wall`, `${root}light`, `${root}break`,
];
beforeEach(() => {
vi.restoreAllMocks();
vi.clearAllMocks();
});
// =============================================================================
// CONSTANTS
// =============================================================================
describe('LLM_DRILL_TYPES', () => {
it('exports all 14 drill types', () => {
expect(LLM_DRILL_TYPES).toEqual([
'word-association',
'story-recall',
'verbal-fluency',
'wit-comeback',
'pun-wordplay',
'compound-chain',
'bridge-word',
'double-meaning',
'idiom-twist',
'what-if',
'alternative-uses',
'story-prompt',
'invention-pitch',
'reframe',
]);
});
});
describe('callAI', () => {
it('passes an explicit effort and source through the shared runner', async () => {
mockApiProvider({ ok: true });
await callAI('Evaluate this attempt.', 'test-provider', 'test-model', 'high', 'meatspace-post-rhetoric-evaluator');
expect(runPromptThroughProvider).toHaveBeenCalledWith(expect.objectContaining({
effort: 'high',
source: 'meatspace-post-rhetoric-evaluator',
}));
});
});
// =============================================================================
// WORD ASSOCIATION
// =============================================================================
describe('generateWordAssociation', () => {
it('returns word-association drill with questions', async () => {
mockApiProvider({ questions: [
{ prompt: 'cathedral', hints: 'architecture' },
{ prompt: 'river', hints: 'nature' },
{ prompt: 'silence', hints: 'abstract' }
]});
const result = await generateWordAssociation({ count: 3 });
expect(result.type).toBe('word-association');
expect(result.config.count).toBe(3);
expect(result.questions).toHaveLength(3);
expect(result.questions[0]).toHaveProperty('prompt', 'cathedral');
expect(result.questions[0]).toHaveProperty('hints', 'architecture');
});
it('defaults count to 5', async () => {
mockApiProvider({ questions: Array.from({ length: 5 }, (_, i) => ({ prompt: `word${i}` })) });
const result = await generateWordAssociation({});
expect(result.config.count).toBe(5);
});
it('rejects a wrong item count instead of truncating the response', async () => {
mockApiProvider({ questions: Array.from({ length: 10 }, (_, i) => ({ prompt: `word${i}` })) });
await expect(generateWordAssociation({ count: 3 })).rejects.toThrow(/questions.*3 item/i);
});
it('defaults hints to empty string', async () => {
mockApiProvider({ questions: [{ prompt: 'test' }] });
const result = await generateWordAssociation({ count: 1 });
expect(result.questions[0].hints).toBe('');
});
it('rejects missing, truncated, and wrong-type output with actionable paths', async () => {
mockApiProvider({ questions: [{ prompt: 42 }] });
await expect(generateWordAssociation({ count: 1 })).rejects.toThrow(/questions\.0\.prompt/);
mockApiProvider({ questions: [] });
await expect(generateWordAssociation({ count: 1 })).rejects.toThrow(/questions/);
mockApiProvider({ prompts: [{ prompt: 'wrong key' }] });
await expect(generateWordAssociation({ count: 1 })).rejects.toThrow(/questions/);
});
it('records only secret-free generation provenance', async () => {
const provider = mockApiProvider({ questions: [{ prompt: 'test' }] });
provider.apiKey = 'do-not-persist';
provider.authHeader = 'do-not-persist';
const result = await generateWordAssociation({ count: 1 });
expect(result.generation).toEqual({
schemaVersion: '1', promptVersion: '1', providerId: 'test-provider', model: 'test-model',
});
expect(JSON.stringify(result.generation)).not.toContain('do-not-persist');
});
});
// =============================================================================
// STORY RECALL
// =============================================================================
describe('generateStoryRecall', () => {
it('returns story-recall drill with exercises', async () => {
mockApiProvider({ exercises: [
{ paragraph: 'A story about Jane...', questions: storyQuestions() }
]});
const result = await generateStoryRecall({ count: 1 });
expect(result.type).toBe('story-recall');
expect(result.config.count).toBe(1);
expect(result.exercises).toHaveLength(1);
expect(result.exercises[0].paragraph).toBe('A story about Jane...');
expect(result.exercises[0].questions[0].answer).toBe('Jane');
});
it('defaults count to 3', async () => {
mockApiProvider({ exercises: Array.from({ length: 3 }, () => ({ paragraph: 'p', questions: storyQuestions() })) });
const result = await generateStoryRecall({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// VERBAL FLUENCY
// =============================================================================
describe('generateVerbalFluency', () => {
it('returns verbal-fluency drill with categories', async () => {
mockApiProvider({ categories: [
{ category: 'Animals', minExpected: 15, examples: ['dog', 'cat', 'otter'] }
]});
const result = await generateVerbalFluency({ count: 1 });
expect(result.type).toBe('verbal-fluency');
expect(result.config.count).toBe(1);
expect(result.categories).toHaveLength(1);
expect(result.categories[0].category).toBe('Animals');
});
it('defaults count to 3', async () => {
mockApiProvider({ categories: Array.from({ length: 3 }, () => ({ category: 'X', minExpected: 10, examples: ['a', 'b', 'c'] })) });
const result = await generateVerbalFluency({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// WIT & COMEBACK
// =============================================================================
describe('generateWitComeback', () => {
it('returns wit-comeback drill with scenarios', async () => {
mockApiProvider({ scenarios: [
{ setup: 'Your friend says...', context: 'at dinner', difficulty: 'medium' }
]});
const result = await generateWitComeback({ count: 1 });
expect(result.type).toBe('wit-comeback');
expect(result.config.count).toBe(1);
expect(result.scenarios).toHaveLength(1);
expect(result.scenarios[0].setup).toBe('Your friend says...');
expect(result.scenarios[0].difficulty).toBe('medium');
});
it('defaults count to 5', async () => {
mockApiProvider({ scenarios: Array.from({ length: 5 }, () => ({ setup: 'x', context: '', difficulty: 'easy' })) });
const result = await generateWitComeback({});
expect(result.config.count).toBe(5);
});
});
// =============================================================================
// PUN & WORDPLAY
// =============================================================================
describe('generatePunWordplay', () => {
it('returns pun-wordplay drill with challenges', async () => {
mockApiProvider({ challenges: [
{ type: 'pun-topic', prompt: 'Make a pun about cats', topic: 'cats', example: 'purr-fect' }
]});
const result = await generatePunWordplay({ count: 1 });
expect(result.type).toBe('pun-wordplay');
expect(result.config.count).toBe(1);
expect(result.challenges).toHaveLength(1);
expect(result.challenges[0].topic).toBe('cats');
});
it('defaults count to 5', async () => {
mockApiProvider({ challenges: Array.from({ length: 5 }, () => ({ type: 'pun-topic', prompt: 'x', topic: 'y', example: 'z' })) });
const result = await generatePunWordplay({});
expect(result.config.count).toBe(5);
});
});
// =============================================================================
// COMPOUND CHAIN
// =============================================================================
describe('generateCompoundChain', () => {
it('returns compound-chain drill with challenges', async () => {
mockApiProvider({ challenges: [
{ rootWord: 'paper', position: 'prefix', examples: compoundExamples('paper'), minExpected: 8 }
]});
const result = await generateCompoundChain({ count: 1 });
expect(result.type).toBe('compound-chain');
expect(result.config.count).toBe(1);
expect(result.challenges).toHaveLength(1);
expect(result.challenges[0].rootWord).toBe('paper');
expect(result.challenges[0].position).toBe('prefix');
expect(result.challenges[0].examples).toEqual(compoundExamples('paper'));
expect(result.challenges[0].minExpected).toBe(8);
});
it('defaults count to 5', async () => {
mockApiProvider({ challenges: Array.from({ length: 5 }, () => ({ rootWord: 'fire', position: 'both', examples: compoundExamples(), minExpected: 8 })) });
const result = await generateCompoundChain({});
expect(result.config.count).toBe(5);
});
it('rejects a partial challenge instead of inventing scoring fields', async () => {
mockApiProvider({ challenges: [{ rootWord: 'light' }] });
await expect(generateCompoundChain({ count: 1 })).rejects.toThrow(/position|examples|minExpected/);
});
it('rejects answer-key examples that contradict the declared root position', async () => {
mockApiProvider({ challenges: [{
rootWord: 'paper', position: 'prefix', examples: ['newspaper', ...compoundExamples('paper').slice(1)], minExpected: 8,
}] });
await expect(generateCompoundChain({ count: 1 })).rejects.toThrow(/declared position/);
});
});
// =============================================================================
// BRIDGE WORD
// =============================================================================
describe('generateBridgeWord', () => {
it('returns bridge-word drill with puzzles', async () => {
mockApiProvider({ puzzles: [
{ clues: ['news___', '___back', '___weight'], answer: 'paper', difficulty: 'easy', hint: 'You write on it' }
]});
const result = await generateBridgeWord({ count: 1 });
expect(result.type).toBe('bridge-word');
expect(result.config.count).toBe(1);
expect(result.puzzles).toHaveLength(1);
expect(result.puzzles[0].clues).toEqual(['news___', '___back', '___weight']);
expect(result.puzzles[0].answer).toBe('paper');
expect(result.puzzles[0].difficulty).toBe('easy');
expect(result.puzzles[0].hint).toBe('You write on it');
});
it('defaults count to 5', async () => {
mockApiProvider({ puzzles: Array.from({ length: 5 }, () => ({ clues: ['a___', '___b', 'c___'], answer: 'x' })) });
const result = await generateBridgeWord({});
expect(result.config.count).toBe(5);
});
it('defaults optional difficulty and hint while retaining required clues', async () => {
mockApiProvider({ puzzles: [{ clues: ['news___', '___back', '___weight'], answer: 'paper' }] });
const result = await generateBridgeWord({ count: 1 });
expect(result.puzzles[0].clues).toHaveLength(3);
expect(result.puzzles[0].difficulty).toBe('medium');
expect(result.puzzles[0].hint).toBe('');
});
});
// =============================================================================
// DOUBLE MEANING
// =============================================================================
describe('generateDoubleMeaning', () => {
it('returns double-meaning drill with challenges', async () => {
mockApiProvider({ challenges: [
{ word: 'bark', meanings: ['tree covering', 'dog sound'], example: 'The bark of the dog echoed off the bark of the tree.', difficulty: 'easy' }
]});
const result = await generateDoubleMeaning({ count: 1 });
expect(result.type).toBe('double-meaning');
expect(result.config.count).toBe(1);
expect(result.challenges).toHaveLength(1);
expect(result.challenges[0].word).toBe('bark');
expect(result.challenges[0].meanings).toEqual(['tree covering', 'dog sound']);
expect(result.challenges[0].difficulty).toBe('easy');
});
it('defaults count to 5', async () => {
mockApiProvider({ challenges: Array.from({ length: 5 }, () => ({ word: 'x', meanings: ['one', 'two'] })) });
const result = await generateDoubleMeaning({});
expect(result.config.count).toBe(5);
});
it('defaults optional example and difficulty while requiring meanings', async () => {
mockApiProvider({ challenges: [{ word: 'scale', meanings: ['measurement', 'fish covering'] }] });
const result = await generateDoubleMeaning({ count: 1 });
expect(result.challenges[0].meanings).toEqual(['measurement', 'fish covering']);
expect(result.challenges[0].example).toBe('');
expect(result.challenges[0].difficulty).toBe('medium');
});
});
// =============================================================================
// IDIOM TWIST
// =============================================================================
describe('generateIdiomTwist', () => {
it('returns idiom-twist drill with challenges', async () => {
mockApiProvider({ challenges: [
{ idiom: "Don't put all your eggs in one basket", domain: 'programming', example: "Don't push all your commits to one branch", difficulty: 'easy' }
]});
const result = await generateIdiomTwist({ count: 1 });
expect(result.type).toBe('idiom-twist');
expect(result.config.count).toBe(1);
expect(result.challenges).toHaveLength(1);
expect(result.challenges[0].idiom).toBe("Don't put all your eggs in one basket");
expect(result.challenges[0].domain).toBe('programming');
expect(result.challenges[0].difficulty).toBe('easy');
});
it('defaults count to 5', async () => {
mockApiProvider({ challenges: Array.from({ length: 5 }, () => ({ idiom: 'x', domain: 'y' })) });
const result = await generateIdiomTwist({});
expect(result.config.count).toBe(5);
});
it('defaults example and difficulty when absent', async () => {
mockApiProvider({ challenges: [{ idiom: 'x', domain: 'y' }] });
const result = await generateIdiomTwist({ count: 1 });
expect(result.challenges[0].example).toBe('');
expect(result.challenges[0].difficulty).toBe('medium');
});
});
// =============================================================================
// WHAT IF
// =============================================================================
describe('generateWhatIf', () => {
it('returns what-if drill with scenarios', async () => {
mockApiProvider({ scenarios: [
{ prompt: 'What if gravity reversed for 10 minutes every Tuesday?', category: 'physics' }
]});
const result = await generateWhatIf({ count: 1 });
expect(result.type).toBe('what-if');
expect(result.config.count).toBe(1);
expect(result.scenarios).toHaveLength(1);
expect(result.scenarios[0].category).toBe('physics');
});
it('defaults count to 3', async () => {
mockApiProvider({ scenarios: Array.from({ length: 3 }, () => ({ prompt: 'x', category: 'y' })) });
const result = await generateWhatIf({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// ALTERNATIVE USES
// =============================================================================
describe('generateAlternativeUses', () => {
it('returns alternative-uses drill with objects', async () => {
mockApiProvider({ objects: [
{ object: 'brick', commonUse: 'building material', minExpected: 8 }
]});
const result = await generateAlternativeUses({ count: 1 });
expect(result.type).toBe('alternative-uses');
expect(result.config.count).toBe(1);
expect(result.objects).toHaveLength(1);
expect(result.objects[0].object).toBe('brick');
expect(result.objects[0].minExpected).toBe(8);
});
it('defaults count to 3', async () => {
mockApiProvider({ objects: Array.from({ length: 3 }, () => ({ object: 'x', commonUse: 'y', minExpected: 5 })) });
const result = await generateAlternativeUses({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// STORY PROMPT
// =============================================================================
describe('generateStoryPrompt', () => {
it('returns story-prompt drill with prompts', async () => {
mockApiProvider({ prompts: [
{ words: ['lighthouse', 'saxophone', 'marmalade'] }
]});
const result = await generateStoryPrompt({ count: 1 });
expect(result.type).toBe('story-prompt');
expect(result.config.count).toBe(1);
expect(result.prompts).toHaveLength(1);
expect(result.prompts[0].words).toEqual(['lighthouse', 'saxophone', 'marmalade']);
});
it('defaults count to 3', async () => {
mockApiProvider({ prompts: Array.from({ length: 3 }, () => ({ words: ['a', 'b', 'c'] })) });
const result = await generateStoryPrompt({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// INVENTION PITCH
// =============================================================================
describe('generateInventionPitch', () => {
it('returns invention-pitch drill with problems', async () => {
mockApiProvider({ problems: [
{ problem: 'You always forget where you put your keys', category: 'everyday', difficulty: 'easy' }
]});
const result = await generateInventionPitch({ count: 1 });
expect(result.type).toBe('invention-pitch');
expect(result.config.count).toBe(1);
expect(result.problems).toHaveLength(1);
expect(result.problems[0].category).toBe('everyday');
expect(result.problems[0].difficulty).toBe('easy');
});
it('defaults count to 3', async () => {
mockApiProvider({ problems: Array.from({ length: 3 }, () => ({ problem: 'x', category: 'y', difficulty: 'medium' })) });
const result = await generateInventionPitch({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// REFRAME
// =============================================================================
describe('generateReframe', () => {
it('returns reframe drill with situations', async () => {
mockApiProvider({ situations: [
{ situation: 'Your flight was delayed by 4 hours', severity: 'medium' }
]});
const result = await generateReframe({ count: 1 });
expect(result.type).toBe('reframe');
expect(result.config.count).toBe(1);
expect(result.situations).toHaveLength(1);
expect(result.situations[0].severity).toBe('medium');
});
it('defaults count to 3', async () => {
mockApiProvider({ situations: Array.from({ length: 3 }, () => ({ situation: 'x', severity: 'low' })) });
const result = await generateReframe({});
expect(result.config.count).toBe(3);
});
});
// =============================================================================
// generateLlmDrill ROUTER
// =============================================================================
describe('generateLlmDrill', () => {
it('routes to correct generator for each type', async () => {
mockApiProvider({ questions: [{ prompt: 'test' }] });
const wa = await generateLlmDrill('word-association', { count: 1 });
expect(wa.type).toBe('word-association');
mockApiProvider({ exercises: [{ paragraph: 'p', questions: storyQuestions() }] });
const sr = await generateLlmDrill('story-recall', { count: 1 });
expect(sr.type).toBe('story-recall');
mockApiProvider({ categories: [{ category: 'X', minExpected: 10, examples: ['a', 'b', 'c'] }] });
const vf = await generateLlmDrill('verbal-fluency', { count: 1 });
expect(vf.type).toBe('verbal-fluency');
mockApiProvider({ scenarios: [{ setup: 'x', context: '', difficulty: 'easy' }] });
const wc = await generateLlmDrill('wit-comeback', { count: 1 });
expect(wc.type).toBe('wit-comeback');
mockApiProvider({ challenges: [{ type: 'pun-topic', prompt: 'x', topic: 'y', example: 'z' }] });
const pw = await generateLlmDrill('pun-wordplay', { count: 1 });
expect(pw.type).toBe('pun-wordplay');
mockApiProvider({ challenges: [{ rootWord: 'fire', position: 'both', examples: compoundExamples(), minExpected: 8 }] });
const cc = await generateLlmDrill('compound-chain', { count: 1 });
expect(cc.type).toBe('compound-chain');
mockApiProvider({ puzzles: [{ clues: ['news___', '___back', '___weight'], answer: 'paper' }] });
const bw = await generateLlmDrill('bridge-word', { count: 1 });
expect(bw.type).toBe('bridge-word');
mockApiProvider({ challenges: [{ word: 'bark', meanings: ['tree covering', 'dog sound'] }] });
const dm = await generateLlmDrill('double-meaning', { count: 1 });
expect(dm.type).toBe('double-meaning');
mockApiProvider({ challenges: [{ idiom: 'x', domain: 'y' }] });
const twist = await generateLlmDrill('idiom-twist', { count: 1 });
expect(twist.type).toBe('idiom-twist');
mockApiProvider({ scenarios: [{ prompt: 'x', category: 'y' }] });
const wi = await generateLlmDrill('what-if', { count: 1 });
expect(wi.type).toBe('what-if');
mockApiProvider({ objects: [{ object: 'x', commonUse: 'y', minExpected: 5 }] });
const au = await generateLlmDrill('alternative-uses', { count: 1 });
expect(au.type).toBe('alternative-uses');
mockApiProvider({ prompts: [{ words: ['a', 'b', 'c'] }] });
const sp = await generateLlmDrill('story-prompt', { count: 1 });
expect(sp.type).toBe('story-prompt');
mockApiProvider({ problems: [{ problem: 'x', category: 'y', difficulty: 'easy' }] });
const ip = await generateLlmDrill('invention-pitch', { count: 1 });
expect(ip.type).toBe('invention-pitch');
mockApiProvider({ situations: [{ situation: 'x', severity: 'low' }] });
const rf = await generateLlmDrill('reframe', { count: 1 });
expect(rf.type).toBe('reframe');
});
it('returns null for unknown type', async () => {
const result = await generateLlmDrill('unknown-type');
expect(result).toBeNull();
});
});
// =============================================================================
// LLM SCORING
// =============================================================================
describe('scoreLlmDrill', () => {
it('returns score with evaluation for word-association', async () => {
mockApiProvider({
overallScore: 75,
scores: [{ score: 80, feedback: 'Good associations' }],
summary: 'Solid performance'
});
const result = await scoreLlmDrill(
'word-association',
{ questions: [{ prompt: 'cathedral', hints: '' }] },
[{ response: 'church spire gothic', responseMs: 3000 }],
120000
);
expect(result.score).toBeGreaterThanOrEqual(0);
expect(result.score).toBeLessThanOrEqual(100);
expect(result.evaluation.overallScore).toBe(75);
expect(result.questions[0].llmScore).toBe(80);
expect(result.questions[0].llmFeedback).toBe('Good associations');
expect(result.evaluation.provenance).toEqual({
generator: { schemaVersion: 'legacy', promptVersion: 'legacy', providerId: 'legacy', model: 'legacy' },
scorer: { kind: 'llm', rubricVersion: '1', providerId: 'test-provider', model: 'test-model' },
});
});
it('combines quality (80%) and speed bonus (20%)', async () => {
mockApiProvider({
overallScore: 100,
scores: [{ score: 100, feedback: 'Perfect' }],
summary: 'Perfect'
});
// Fast response: 1s out of 120s limit -> high speed bonus
const fast = await scoreLlmDrill(
'word-association',
{ questions: [{ prompt: 'test' }] },
[{ response: 'answer', responseMs: 1000 }],
120000
);
mockApiProvider({
overallScore: 100,
scores: [{ score: 100, feedback: 'Perfect' }],
summary: 'Perfect'
});
// Slow response: 119s out of 120s -> near-zero speed bonus
const slow = await scoreLlmDrill(
'word-association',
{ questions: [{ prompt: 'test' }] },
[{ response: 'answer', responseMs: 119000 }],
120000
);
expect(fast.score).toBeGreaterThan(slow.score);
});
it('rejects out-of-range scorer output instead of clamping it', async () => {
mockApiProvider({
overallScore: 150,
scores: [{ score: 200, feedback: 'Over max' }],
summary: 'Overcapped'
});
await expect(scoreLlmDrill(
'word-association',
{ questions: [{ prompt: 'test' }] },
[{ response: 'answer', responseMs: 1000 }],
120000
)).rejects.toThrow(/overallScore|score/);
});
it('rejects an unknown drill type instead of fabricating zero', async () => {
await expect(scoreLlmDrill('unknown', {}, [{ response: 'x' }], 60000))
.rejects.toThrow('Unsupported POST LLM scorer type');
});
it('attaches per-response llmScore and llmFeedback', async () => {
mockApiProvider({
overallScore: 60,
scores: [
{ score: 80, feedback: 'Clever' },
{ score: 40, feedback: 'Needs work' }
],
summary: 'Mixed'
});
const result = await scoreLlmDrill(
'wit-comeback',
{ scenarios: [{ setup: 'a' }, { setup: 'b' }] },
[
{ response: 'zinger', responseMs: 2000 },
{ response: 'meh', responseMs: 5000 }
],
120000
);
expect(result.questions).toHaveLength(2);
expect(result.questions[0].llmScore).toBe(80);
expect(result.questions[1].llmScore).toBe(40);
expect(result.questions[1].llmFeedback).toBe('Needs work');
});
it('rejects a missing scores array instead of storing partial feedback', async () => {
mockApiProvider({
overallScore: 50,
summary: 'No per-item scores'
});
await expect(scoreLlmDrill(
'pun-wordplay',
{ challenges: [{ prompt: 'x' }] },
[{ response: 'y', responseMs: 1000 }],
120000
)).rejects.toThrow(/scores/);
});
it('rejects truncated and wrong-type score entries', async () => {
mockApiProvider({
overallScore: 50,
scores: [{ score: '50', feedback: 'wrong type' }],
summary: 'Malformed',
});
await expect(scoreLlmDrill(
'word-association',
{ questions: [{ prompt: 'a' }, { prompt: 'b' }] },
[{ response: 'x' }, { response: 'y' }],
120000,
)).rejects.toThrow(/scores/);
});
it('scores story-recall with answers', async () => {
// story-recall uses local scoring: 1/1 correct = 100
const result = await scoreLlmDrill(
'story-recall',
{ exercises: [{ paragraph: 'Jane went to Paris on Monday.', questions: [{ question: 'Where?', answer: 'Paris' }] }] },
[{ answers: ['Paris'], responseMs: 5000 }],
180000
);
expect(result.score).toBeGreaterThan(0);
expect(result.evaluation.overallScore).toBe(100);
});
it('accepts declared story aliases but rejects substring fragments', async () => {
const drillData = {
exercises: [{
questions: [
{ question: 'Where?', answer: 'New York City', aliases: ['NYC'] },
{ question: 'Who?', answer: 'Alexandra', aliases: [] },
],
}],
};
const result = await scoreLlmDrill(
'story-recall', drillData, [{ answers: ['nyc', 'Alex'], responseMs: 5000 }], 180000,
);
expect(result.evaluation.scores[0]).toMatchObject({ score: 50, feedback: '1 of 2 correct' });
});
it('scores verbal-fluency from finite known examples without a provider call', async () => {
const animals = ['dog', 'cat', 'fish', 'bird', 'snake', 'lion', 'tiger', 'bear', 'wolf', 'fox'];
const result = await scoreLlmDrill(
'verbal-fluency',
{ categories: [{ category: 'Animals', minExpected: 15, examples: animals }] },
[{ items: animals, responseMs: 45000 }],
60000
);
expect(result.score).toBeGreaterThan(0);
expect(result.evaluation.overallScore).toBe(67);
expect(runPromptThroughProvider).not.toHaveBeenCalled();
});
it('batch-validates verbal items once and separates valid, invalid, and duplicate items', async () => {
mockApiProvider({
verdicts: [
{ responseIndex: 0, itemIndex: 1, valid: true },
{ responseIndex: 0, itemIndex: 4, valid: false },
],
});
const result = await scoreLlmDrill(
'verbal-fluency',
{ categories: [{ category: 'Animals', minExpected: 3, examples: ['dog'] }] },
[{ items: ['dog', 'otter', 'OTTER', '', 'not an animal category'], responseMs: 10000 }],
60000,
);
expect(runPromptThroughProvider).toHaveBeenCalledTimes(1);
expect(result.evaluation.scores[0]).toMatchObject({
validCount: 2,
validItems: ['dog', 'otter'],
invalidItems: ['', 'not an animal category'],
duplicateItems: ['OTTER'],
});
});
it('compound-chain accepts both full compounds and the other half', async () => {
// User-typed shorthand: "hose" instead of "firehose", "pit" instead of "firepit".
// Both should count as valid compound contributions.
const result = await scoreLlmDrill(
'compound-chain',
{ challenges: [{ rootWord: 'fire', position: 'both', minExpected: 4, examples: ['firehose', 'firepit', 'firework', 'campfire'] }] },
[{ items: ['hose', 'pit', 'work', 'campfire'], responseMs: 30000 }],
60000
);
expect(result.evaluation.scores[0].validCount).toBe(4);
expect(result.evaluation.scores[0].invalidItems).toEqual([]);
// Examples already covered (either as full compound or half) shouldn't be re-suggested.
expect(result.evaluation.scores[0].missedExamples).not.toContain('firehose');
expect(result.evaluation.scores[0].missedExamples).not.toContain('firepit');
expect(result.evaluation.scores[0].missedExamples).not.toContain('firework');
expect(result.evaluation.scores[0].missedExamples).not.toContain('campfire');
});
it('compound-chain rejects bare root word', () => {
return scoreLlmDrill(
'compound-chain',
{ challenges: [{ rootWord: 'fire', position: 'both', minExpected: 2, examples: ['firework'] }] },
[{ items: ['fire', 'firework'], responseMs: 10000 }],
60000
).then(result => {
expect(result.evaluation.scores[0].validCount).toBe(1);
expect(result.evaluation.scores[0].invalidItems).toContain('fire');
});
});
it('batch-validates unknown compound fragments exactly once', async () => {
mockApiProvider({
verdicts: [
{ responseIndex: 0, itemIndex: 1, valid: true },
{ responseIndex: 0, itemIndex: 3, valid: false },
{ responseIndex: 0, itemIndex: 4, valid: true },
],
});
const result = await scoreLlmDrill(
'compound-chain',
{ challenges: [{ rootWord: 'fire', position: 'both', minExpected: 3, examples: [] }] },
[{ items: ['fire', 'firehouse', 'FIRE HOUSE', 'firebanana', 'wildfire'], responseMs: 10000 }],
60000,
);
expect(runPromptThroughProvider).toHaveBeenCalledTimes(1);
expect(result.evaluation.scores[0]).toMatchObject({
validCount: 2,
validItems: ['firehouse', 'wildfire'],
invalidItems: ['fire', 'firebanana'],
duplicateItems: ['FIRE HOUSE'],
});
});
it('rejects incomplete semantic verdict batches', async () => {
mockApiProvider({ verdicts: [] });
await expect(scoreLlmDrill(
'compound-chain',
{ challenges: [{ rootWord: 'fire', position: 'both', minExpected: 1, examples: [] }] },
[{ items: ['firehouse'], responseMs: 1000 }],
60000,
)).rejects.toThrow(/verdicts/);
});
});
// =============================================================================
// LOCAL SCORING — BRIDGE WORD
// =============================================================================
describe('scoreLlmDrill bridge-word (local scoring)', () => {
it('scores 100 for an exact match (case/whitespace insensitive)', async () => {
const result = await scoreLlmDrill(
'bridge-word',
{ puzzles: [{ clues: ['news___', '___back'], answer: 'paper' }] },
[{ response: ' Paper ', responseMs: 5000 }],
60000
);
expect(result.evaluation.scores[0].score).toBe(100);
expect(result.evaluation.scores[0].feedback).toBe('Correct!');
expect(result.evaluation.overallScore).toBe(100);
});
it('scores 0 for a wrong answer with "the answer was" feedback', async () => {
const result = await scoreLlmDrill(
'bridge-word',
{ puzzles: [{ clues: ['news___', '___back'], answer: 'paper' }] },
[{ response: 'wood', responseMs: 5000 }],
60000
);
expect(result.evaluation.scores[0].score).toBe(0);
expect(result.evaluation.scores[0].feedback).toBe('The answer was "paper"');
});
it('rejects a missing answer key instead of fabricating a zero score', async () => {
await expect(scoreLlmDrill(
'bridge-word',
{ puzzles: [] },
[{ response: 'paper', responseMs: 5000 }],
60000
)).rejects.toThrow(/missing answer key/);
});
});
// =============================================================================
// LLM SCORING — DOUBLE MEANING / IDIOM TWIST
// =============================================================================
describe('scoreLlmDrill double-meaning and idiom-twist (LLM scoring)', () => {
it('scores double-meaning with evaluation and per-response feedback', async () => {
mockApiProvider({
overallScore: 85,
scores: [{ score: 90, feedback: 'Both meanings used cleverly' }],
summary: 'Great wordplay'
});
const result = await scoreLlmDrill(
'double-meaning',
{ challenges: [{ word: 'bark', meanings: ['tree covering', 'dog sound'] }] },
[{ response: "The dog's bark echoed off the bark of the tree.", responseMs: 8000 }],
60000
);
expect(result.evaluation.overallScore).toBe(85);
expect(result.questions[0].llmScore).toBe(90);
expect(result.questions[0].llmFeedback).toBe('Both meanings used cleverly');
});
it('scores idiom-twist with evaluation and per-response feedback', async () => {
mockApiProvider({
overallScore: 70,
scores: [{ score: 75, feedback: 'Good structural parallel' }],
summary: 'Solid twist'
});
const result = await scoreLlmDrill(
'idiom-twist',
{ challenges: [{ idiom: "Don't put all your eggs in one basket", domain: 'programming' }] },
[{ response: "Don't push all your commits to one branch", responseMs: 8000 }],
60000
);
expect(result.evaluation.overallScore).toBe(70);
expect(result.questions[0].llmScore).toBe(75);
expect(result.questions[0].llmFeedback).toBe('Good structural parallel');
});
});
// =============================================================================
// PROVIDER SELECTION
// =============================================================================
describe('provider selection', () => {
it('uses active provider when no providerId given', async () => {
mockApiProvider({ questions: [{ prompt: 'test' }] });
await generateWordAssociation({ count: 1 });
expect(getActiveProvider).toHaveBeenCalled();
});
it('uses specific provider when providerId given', async () => {
mockApiProvider({ questions: [{ prompt: 'test' }] });
await generateWordAssociation({ count: 1 }, 'specific-provider');
expect(getProviderById).toHaveBeenCalledWith('specific-provider');
});
it('throws when no provider is available', async () => {
getActiveProvider.mockResolvedValue(null);
await expect(generateWordAssociation({ count: 1 })).rejects.toThrow('No AI provider available');
});
it('throws when provider is disabled', async () => {
getActiveProvider.mockResolvedValue({ id: 'test', enabled: false, type: 'api' });
await expect(generateWordAssociation({ count: 1 })).rejects.toThrow('No AI provider available');
});
});
// =============================================================================
// JSON PARSING ROBUSTNESS
// =============================================================================
describe('AI response parsing', () => {
it('handles markdown-fenced JSON', async () => {
const provider = {
id: 'test', enabled: true, type: 'api',
endpoint: 'http://localhost:9999', defaultModel: 'test'
};
getActiveProvider.mockResolvedValue(provider);
runPromptThroughProvider.mockResolvedValue({
text: '```json\n{"questions":[{"prompt":"hello"}]}\n```',
runId: 'test-run', model: 'test-model'
});
const result = await generateWordAssociation({ count: 1 });
expect(result.questions[0].prompt).toBe('hello');
});
it('handles JSON with surrounding text', async () => {
const provider = {
id: 'test', enabled: true, type: 'api',
endpoint: 'http://localhost:9999', defaultModel: 'test'
};
getActiveProvider.mockResolvedValue(provider);
runPromptThroughProvider.mockResolvedValue({
text: 'Here is the result:\n{"questions":[{"prompt":"world"}]}\nHope this helps!',
runId: 'test-run', model: 'test-model'
});
const result = await generateWordAssociation({ count: 1 });
expect(result.questions[0].prompt).toBe('world');
});
it('skips an echoed generation schema before the real response', async () => {
const provider = {
id: 'test', enabled: true, type: 'api',
endpoint: 'http://localhost:9999', defaultModel: 'test'
};
getActiveProvider.mockResolvedValue(provider);
runPromptThroughProvider.mockImplementation(async ({ prompt }) => ({
// Some CLI providers echo the complete prompt, including its valid
// looking schema example, before printing the actual answer.
text: `${prompt}\n${JSON.stringify({
questions: [{ prompt: 'real-word', hints: 'real hint' }]
})}`,
runId: 'test-run', model: 'test-model'
}));
const result = await generateWordAssociation({ count: 1 });
expect(result.questions[0]).toEqual({ prompt: 'real-word', hints: 'real hint' });
});
it('throws on empty AI response', async () => {
const provider = {
id: 'test', enabled: true, type: 'api',
endpoint: 'http://localhost:9999', defaultModel: 'test'
};
getActiveProvider.mockResolvedValue(provider);
runPromptThroughProvider.mockResolvedValue({
text: '', runId: 'test-run', model: 'test-model'
});
await expect(generateWordAssociation({ count: 1 })).rejects.toThrow();
});
});