Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(router): attribute fallback learning to each attempted model

Rudy Celekli3b3681480cdf
diff --git a/v3/@claude-flow/cli/__tests__/agent-fallback-feedback.test.ts b/v3/@claude-flow/cli/__tests__/agent-fallback-feedback.test.ts
new file mode 100644
index 0000000000..8b009b2296
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/agent-fallback-feedback.test.ts
@@ -0,0 +1,55 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { executeAgentTask } from '../src/mcp-tools/agent-execute-core.js';
+const mocks = vi.hoisted(() => ({ next: vi.fn(), tier: vi.fn(), model: vi.fn() }));
+vi.mock('../src/ruvector/neural-router.js', () => ({ nextCostOptimalAlternative: mocks.next }));
+vi.mock('../src/ruvector/task-embedder.js', () => ({ embedTaskWithCache: async () => [1, 0] }));
+vi.mock('../src/ruvector/model-router.js', () => ({ recordModelOutcome: mocks.tier, recordModelOutcomeByModelId: mocks.model }));
+const failed = {ok: false, status: 503, text: async () => 'unavailable'};
+const success = {ok: true, json: async () => ({model: 'openai/winner', choices: [{message: {content: 'done'}}]})};
+
+describe('fallback execution learning attribution', () => {
+ let dir: string;
+ let storePath: string;
+ let fetchMock: ReturnType<typeof vi.fn>;
+ beforeEach(() => {
+ vi.resetAllMocks();
+ dir = mkdtempSync(join(tmpdir(), 'ruflo-fallback-feedback-'));
+ for (const key of ['ANTHROPIC_API_KEY', 'ANTHROPIC_AUTH_TOKEN', 'ANTHROPIC_BASE_URL', 'OLLAMA_API_KEY', 'OLLAMA_BASE_URL', 'RUFLO_PROVIDER', 'OPENROUTER_BASE_URL', 'CLAUDE_FLOW_ROUTER_TRAJECTORY', 'CLAUDE_FLOW_RUN_TRANSCRIPTS']) vi.stubEnv(key, '');
+ vi.stubEnv('OPENROUTER_API_KEY', 'test'); vi.stubEnv('CLAUDE_FLOW_CWD', dir);
+ vi.stubEnv('CLAUDE_FLOW_ROUTER_FALLBACK_MAX_RETRIES', '2');
+ const agents = join(dir, '.claude-flow', 'agents'); mkdirSync(agents, {recursive: true}); storePath = join(agents, 'store.json');
+ writeFileSync(storePath, JSON.stringify({version: '3.0.0', agents: {test: {
+ agentId: 'test', agentType: 'coder', status: 'idle', health: 100, taskCount: 0, config: {}, model: 'opus-4.7', modelId: 'openai/first',
+ }}}));
+ mocks.next.mockResolvedValueOnce({modelId: 'openai/second', model: 'sonnet'}).mockResolvedValueOnce({modelId: 'openai/winner', model: 'haiku'});
+ fetchMock = vi.fn().mockResolvedValue(failed);
+ vi.stubGlobal('fetch', fetchMock);
+ });
+ afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); rmSync(dir, {recursive: true, force: true}); });
+
+ it('credits the successful fallback tier and keeps both failed-model outcomes', async () => {
+ fetchMock.mockResolvedValueOnce(failed).mockResolvedValueOnce(failed).mockResolvedValueOnce(success);
+ const result = await executeAgentTask({agentId: 'test', prompt: 'task'});
+ expect(result.success).toBe(true);
+ expect(mocks.tier.mock.calls).toEqual([['task', 'opus', 'failure'], ['task', 'sonnet', 'failure'], ['task', 'haiku', 'success']]);
+ expect(mocks.model.mock.calls).toEqual([['task', 'openai/first', 'failure'], ['task', 'openai/second', 'failure'], ['task', 'openai/winner', 'success']]);
+ expect(JSON.parse(readFileSync(storePath, 'utf8')).agents.test).toMatchObject({model: 'haiku', modelId: 'openai/winner'});
+ });
+
+ it('records each failed attempted model exactly once when the retry budget is exhausted', async () => {
+ expect((await executeAgentTask({agentId: 'test', prompt: 'task'})).success).toBe(false);
+ expect(mocks.tier.mock.calls).toEqual([['task', 'opus', 'failure'], ['task', 'sonnet', 'failure'], ['task', 'haiku', 'failure']]);
+ expect(mocks.model.mock.calls).toEqual([['task', 'openai/first', 'failure'], ['task', 'openai/second', 'failure'], ['task', 'openai/winner', 'failure']]);
+ });
+
+ it('does not invent a second outcome when no alternative can be executed', async () => {
+ mocks.next.mockReset().mockResolvedValue(null);
+ await executeAgentTask({agentId: 'test', prompt: 'task'});
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(mocks.tier.mock.calls).toEqual([['task', 'opus', 'failure']]);
+ expect(mocks.model.mock.calls).toEqual([['task', 'openai/first', 'failure']]);
+ });
+});
diff --git a/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts b/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts
index 4debea7890..134355a751 100644
--- a/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts
+++ b/v3/@claude-flow/cli/src/mcp-tools/agent-execute-core.ts
@@ -629,6 +629,8 @@ export async function executeAgentTask(input: AgentExecuteInput): Promise<AgentE
// upstream outages don't cause retry storms.
const fallbackBudget = Math.max(0, parseInt(process.env.CLAUDE_FLOW_ROUTER_FALLBACK_MAX_RETRIES ?? '1', 10) || 1);
const fallbackHistory: Array<{ modelId: string; error: string }> = [];
+ // Keep feedback for each completed attempt before switching model identity.
+ const failedAttempts: Array<{ model: ClaudeModel; modelId: string }> = [];
if (!result.success && agent.modelId && fallbackBudget > 0) {
const isRetryable = /\b(429|500|502|503|504|timeout|ECONNRESET|ETIMEDOUT)\b/i.test(result.error ?? '');
if (isRetryable) {
@@ -657,7 +659,9 @@ export async function executeAgentTask(input: AgentExecuteInput): Promise<AgentE
});
// Record the model that ACTUALLY answered (or errored). On success,
// update agent.modelId so downstream observers see the retry winner.
+ failedAttempts.push({ model: agent.model ?? 'sonnet', modelId: agent.modelId! });
agent.modelId = alt.modelId;
+ agent.model = alt.model;
result = altResult;
}
}
@@ -685,6 +689,10 @@ export async function executeAgentTask(input: AgentExecuteInput): Promise<AgentE
agent.model === 'opus-4.7' ? 'opus' :
(agent.model as 'haiku' | 'sonnet' | 'opus' | 'inherit' | undefined) ?? 'sonnet';
const outcome: 'success' | 'failure' = result.success ? 'success' : 'failure';
+ for (const attempt of failedAttempts) {
+ recordModelOutcome(input.prompt, attempt.model === 'opus-4.7' ? 'opus' : attempt.model, 'failure');
+ recordModelOutcomeByModelId(input.prompt, attempt.modelId, 'failure');
+ }
recordModelOutcome(input.prompt, tier, outcome);
// ADR-149 — also write to the shadow per-modelId priors when the cost-
// optimal neural backend picked a concrete model id. Selection logic