🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated
TypeScript agentic-aiagentic-frameworkagentic-workflowagentsai-agentsai-assistantai-skillsautonomous-agentsclaudecodexdsh-pluginharnessmcp-servermulti-agentmulti-agent-systemsnpmskillsswarmswarm-intelligencetypescript
fix(router): attribute fallback learning to each attempted model
Rudy Celekli3b3681480cdf
diff --git a/v3/@claude-flow/cli/__tests__/agent-fallback-feedback.test.ts b/v3/@claude-flow/cli/__tests__/agent-fallback-feedback.test.tsnew file mode 100644index 0000000000..8b009b2296--- /dev/null+++ b/v3/@claude-flow/cli/__tests__/agent-fallback-feedback.test.ts@@ -0,0 +1,55 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { executeAgentTask } from '../src/mcp-tools/agent-execute-core.js';+const mocks = vi.hoisted(() => ({ next: vi.fn(), tier: vi.fn(), model: vi.fn() }));+vi.mock('../src/ruvector/neural-router.js', () => ({ nextCostOptimalAlternative: mocks.next }));+vi.mock('../src/ruvector/task-embedder.js', () => ({ embedTaskWithCache: async () => [1, 0] }));+vi.mock('../src/ruvector/model-router.js', () => ({ recordModelOutcome: mocks.tier, recordModelOutcomeByModelId: mocks.model }));+const failed = {ok: false, status: 503, text: async () => 'unavailable'};+const success = {ok: true, json: async () => ({model: 'openai/winner', choices: [{message: {content: 'done'}}]})};++describe('fallback execution learning attribution', () => {+ let dir: string;+ let storePath: string;+ let fetchMock: ReturnType<typeof vi.fn>;+ beforeEach(() => {+ vi.resetAllMocks();+ dir = mkdtempSync(join(tmpdir(), 'ruflo-fallback-feedback-'));+ for (const key of ['ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_BASE_URL', 'OLLAMA_API_KEY', 'OLLAMA_BASE_URL', 'RUFLO_PROVIDER', 'OPENROUTER_BASE_URL', 'CLAUDE_FLOW_ROUTER_TRAJECTORY', 'CLAUDE_FLOW_RUN_TRANSCRIPTS']) vi.stubEnv(key, '');+ vi.stubEnv('OPENROUTER_API_KEY', 'test'); vi.stubEnv('CLAUDE_FLOW_CWD', dir);+ vi.stubEnv('CLAUDE_FLOW_ROUTER_FALLBACK_MAX_RETRIES', '2');+ const agents = join(dir, '.claude-flow', 'agents'); mkdirSync(agents, {recursive: true}); storePath = join(agents, 'store.json');+ writeFileSync(storePath, JSON.stringify({version: '3.0.0', agents: {test: {+ agentId: 'test', agentType: 'coder', status: 'idle', health: 100, taskCount: 0, config: {}, model: 'opus-4.7', modelId: 'openai/first',+ }}}));+ mocks.next.mockResolvedValueOnce({modelId: 'openai/second', model: 'sonnet'}).mockResolvedValueOnce({modelId: 'openai/winner', model: 'haiku'});+ fetchMock = vi.fn().mockResolvedValue(failed);+ vi.stubGlobal('fetch', fetchMock);+ });+ afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); rmSync(dir, {recursive: true, force: true}); });++ it('credits the successful fallback tier and keeps both failed-model outcomes', async () => {+ fetchMock.mockResolvedValueOnce(failed).mockResolvedValueOnce(failed).mockResolvedValueOnce(success);+ const result = await executeAgentTask({agentId: 'test', prompt: 'task'});+ expect(result.success).toBe(true);+ expect(mocks.tier.mock.calls).toEqual([['task', 'opus', 'failure'], ['task', 'sonnet', 'failure'], ['task', 'haiku', 'success']]);+ expect(mocks.model.mock.calls).toEqual([['task', 'openai/first', 'failure'], ['task', 'openai/second', 'failure'], ['task', 'openai/winner', 'success']]);+ expect(JSON.parse(readFileSync(storePath, 'utf8')).agents.test).toMatchObject({model: 'haiku', modelId: 'openai/winner'});+ });++ it('records each failed attempted model exactly once when the retry budget is exhausted', async () => {+ expect((await executeAgentTask({agentId: 'test', prompt: 'task'})).success).toBe(false);+ expect(mocks.tier.mock.calls).toEqual([['task', 'opus', 'failure'], ['task', 'sonnet', 'failure'], ['task', 'haiku', 'failure']]);+ expect(mocks.model.mock.calls).toEqual([['task', 'openai/first', 'failure'], ['task', 'openai/second', 'failure'], ['task', 'openai/winner', 'failure']]);+ });++ it('does not invent a second outcome when no alternative can be executed', async () => {+ mocks.next.mockReset().mockResolvedValue(null);+ await executeAgentTask({agentId: 'test', prompt: 'task'});+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(mocks.tier.mock.calls).toEqual([['task', 'opus', 'failure']]);+ expect(mocks.model.mock.calls).toEqual([['task', 'openai/first', 'failure']]);+ });+});diff --git a/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts b/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.tsindex 4debea7890..134355a751 100644--- a/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts+++ b/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts@@ -629,6 +629,8 @@ export async function executeAgentTask(input: AgentExecuteInput): Promise<AgentE// upstream outages don't cause retry storms.const fallbackBudget = Math.max(0, parseInt(process.env.CLAUDE_FLOW_ROUTER_FALLBACK_MAX_RETRIES ?? '1', 10) || 1);const fallbackHistory: Array<{ modelId: string; error: string }> = [];+ // Keep feedback for each completed attempt before switching model identity.+ const failedAttempts: Array<{ model: ClaudeModel; modelId: string }> = [];if (!result.success && agent.modelId && fallbackBudget > 0) {const isRetryable = /\b(429|500|502|503|504|timeout|ECONNRESET|ETIMEDOUT)\b/i.test(result.error ?? '');if (isRetryable) {@@ -657,7 +659,9 @@ export async function executeAgentTask(input: AgentExecuteInput): Promise<AgentE});// Record the model that ACTUALLY answered (or errored). On success,// update agent.modelId so downstream observers see the retry winner.+ failedAttempts.push({ model: agent.model ?? 'sonnet', modelId: agent.modelId! });agent.modelId = alt.modelId;+ agent.model = alt.model;result = altResult;}}@@ -685,6 +689,10 @@ export async function executeAgentTask(input: AgentExecuteInput): Promise<AgentEagent.model === 'opus-4.7' ? 'opus' :(agent.model as 'haiku' | 'sonnet' | 'opus' | 'inherit' | undefined) ?? 'sonnet';const outcome: 'success' | 'failure' = result.success ? 'success' : 'failure';+ for (const attempt of failedAttempts) {+ recordModelOutcome(input.prompt, attempt.model === 'opus-4.7' ? 'opus' : attempt.model, 'failure');+ recordModelOutcomeByModelId(input.prompt, attempt.modelId, 'failure');+ }recordModelOutcome(input.prompt, tier, outcome);// ADR-149 — also write to the shadow per-modelId priors when the cost-// optimal neural backend picked a concrete model id. Selection logic