Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { mkdir, rm, writeFile } from 'fs/promises';
import { join } from 'path';
import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest';
const mockCosState = vi.hoisted(() => ({
// Use $TMPDIR (falls back to /tmp) rather than a hardcoded /private/tmp — the
// latter exists on macOS (where /tmp symlinks to it) but not on Linux CI.
agentsDir: `${process.env.TMPDIR || process.env.TEMP || process.env.TMP || '/tmp'}/portos-cos-feedback-test-${process.pid}`,
state: null
}));
const mockAgentIndex = vi.hoisted(() => ({ entries: new Map() }));
vi.mock('./cosState.js', () => ({
AGENTS_DIR: mockCosState.agentsDir,
loadState: vi.fn(async () => mockCosState.state),
saveState: vi.fn(),
withStateLock: async (fn) => fn()
}));
vi.mock('./cosAgentIndex.js', () => ({
loadAgentIndex: vi.fn(async () => mockAgentIndex.entries),
getAgentDir: (agentId, dateBucket) => join(mockCosState.agentsDir, dateBucket || '', agentId)
}));
// The operator-action ledger's file backend writes under PATHS.data — re-root it
// at a temp dir so this suite never touches the developer's live `data/` tree
// (#3683/#3687). Everything else in fileUtils stays real (agent metadata I/O).
const ledgerRoot = await vi.hoisted(async () => {
const { mkdtempSync } = await import('node:fs');
const { tmpdir } = await import('node:os');
const { join: joinPath } = await import('node:path');
return mkdtempSync(joinPath(tmpdir(), 'portos-agent-feedback-ledger-'));
});
vi.mock('../lib/fileUtils.js', async () => {
const actual = await vi.importActual('../lib/fileUtils.js');
const { makePathsProxy } = await import('../lib/mockPathsDataRoot.js');
return makePathsProxy(actual, { dataRoot: ledgerRoot });
});
import {
getFeedbackStats,
getPendingAgentFeedback,
getPendingAgentFeedbackCount,
getPendingAgentFeedbackPage,
initializeAgentFeedback,
submitAgentFeedback,
} from './cosAgentFeedback.js';
import { encodeCompletionOrder } from '../lib/cosAgentCompletionOrder.js';
import { resetCompletionOrderIndex } from './cosAgentCompletionIndex.js';
import { cosEvents } from './cosEvents.js';
import {
listPendingAgentFeedbackRefs,
resetPendingAgentFeedbackStore,
} from './cosAgentFeedbackStore.js';
import { listUserActions } from './userActions.js';
describe('getPendingAgentFeedbackCount', () => {
beforeEach(async () => {
mockCosState.state = { agents: {} };
mockAgentIndex.entries = new Map();
resetPendingAgentFeedbackStore();
await rm(join(ledgerRoot, 'cos-pending-agent-feedback.json'), { force: true });
});
afterAll(async () => {
await rm(mockCosState.agentsDir, { recursive: true, force: true });
});
it('counts only unrated completed non-system manually-run agents for the feedback insight', async () => {
mockCosState.state.agents = {
'agent-unrated': { id: 'agent-unrated', status: 'completed', completedAt: '2026-08-01T10:00:00.000Z', metadata: { taskType: 'user' } },
'agent-rated': { id: 'agent-rated', status: 'completed', metadata: { taskType: 'user' }, feedback: { rating: 'positive' } },
'agent-system': { id: 'agent-system', taskId: 'sys-health-check', status: 'completed', metadata: { taskType: 'user' } },
'agent-running': { id: 'agent-running', status: 'running', metadata: { taskType: 'user' } },
'agent-scheduled': { id: 'agent-scheduled', status: 'completed', metadata: { taskType: 'internal' } }
};
await expect(getPendingAgentFeedbackCount()).resolves.toBe(1);
});
it('excludes system agents identified by id as well as taskId', async () => {
mockCosState.state.agents = {
'sys-nightly-sweep': { id: 'sys-nightly-sweep', status: 'completed', metadata: { taskType: 'user' } },
'agent-real': { id: 'agent-real', status: 'completed', metadata: { taskType: 'user' } }
};
await expect(getPendingAgentFeedbackCount()).resolves.toBe(1);
});
it('excludes scheduled-task and autopilot agents (taskType internal) from the feedback ask', async () => {
mockCosState.state.agents = {
'agent-scheduled': { id: 'agent-scheduled', status: 'completed', metadata: { taskType: 'internal' } },
'agent-manual': { id: 'agent-manual', status: 'completed', metadata: { taskType: 'user' } }
};
await expect(getPendingAgentFeedbackCount()).resolves.toBe(1);
});
it('excludes a run retired by Relaunch, which has no result to rate', async () => {
// A permanent banner otherwise: the count included the record Relaunch
// retires, but AgentCard shows no rating buttons for it, so nothing the user
// could do would ever clear "1 completed run needs feedback".
mockCosState.state.agents = {
'agent-relaunched': {
id: 'agent-relaunched', status: 'completed', completedAt: '2026-08-01T10:00:00.000Z',
metadata: { taskType: 'user' },
result: { success: false, resumed: true, error: 'Relaunched by user on codex' }
},
'agent-manual': { id: 'agent-manual', status: 'completed', metadata: { taskType: 'user' }, result: { success: true } }
};
await expect(getPendingAgentFeedbackCount()).resolves.toBe(1);
});
it('returns 0 when nothing is awaiting a rating', async () => {
await expect(getPendingAgentFeedbackCount()).resolves.toBe(0);
});
it('enrolls a future completion without reading historical archives', async () => {
initializeAgentFeedback();
cosEvents.emit('agent:completed', {
id: 'agent-future',
status: 'completed',
completedAt: '2026-08-02T10:00:00.000Z',
metadata: { taskType: 'user' },
});
await expect.poll(() => listPendingAgentFeedbackRefs()).toEqual([
{ agentId: 'agent-future', archiveDate: '2026-08-02' },
]);
});
it('keeps an eligible run actionable after live state is evicted', async () => {
const agent = {
id: 'agent-archived-pending',
status: 'completed',
completedAt: '2026-08-03T10:00:00.000Z',
metadata: { taskType: 'user', taskDescription: 'Review an example change' },
};
mockCosState.state.agents = { [agent.id]: agent };
await expect(getPendingAgentFeedback()).resolves.toMatchObject({ count: 1, agents: [agent] });
const dir = join(mockCosState.agentsDir, '2026-08-03', agent.id);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'metadata.json'), JSON.stringify(agent));
mockCosState.state = { agents: {} };
mockAgentIndex.entries.set(agent.id, '2026-08-03');
await expect(getPendingAgentFeedback()).resolves.toMatchObject({ count: 1, agents: [agent] });
});
it('retains a deleted reference as unavailable history instead of claiming completion', async () => {
const agent = {
id: 'agent-deleted',
status: 'completed',
completedAt: '2026-08-04T10:00:00.000Z',
metadata: { taskType: 'user' },
};
mockCosState.state.agents = { [agent.id]: agent };
await getPendingAgentFeedback();
mockCosState.state = { agents: {} };
await expect(getPendingAgentFeedback({ includeUnavailable: true })).resolves.toMatchObject({
count: 0,
unavailable: [{ agentId: agent.id, unavailableReason: 'deleted' }],
});
await expect(listPendingAgentFeedbackRefs()).resolves.toEqual([
{ agentId: agent.id, archiveDate: '2026-08-04' },
]);
});
it('retains a reference when the indexed archive cannot be read', async () => {
const agent = {
id: 'agent-unreadable',
status: 'completed',
completedAt: '2026-08-05T10:00:00.000Z',
metadata: { taskType: 'user' },
};
mockCosState.state.agents = { [agent.id]: agent };
await getPendingAgentFeedback();
mockCosState.state = { agents: {} };
mockAgentIndex.entries.set(agent.id, '2026-08-05');
await mkdir(join(mockCosState.agentsDir, '2026-08-05', agent.id, 'metadata.json'), { recursive: true });
await expect(getPendingAgentFeedback({ includeUnavailable: true })).resolves.toMatchObject({
count: 0,
unavailable: [{ agentId: agent.id, unavailableReason: 'read-failed' }],
});
await expect(listPendingAgentFeedbackRefs()).resolves.toEqual([
{ agentId: agent.id, archiveDate: '2026-08-05' },
]);
});
it('clears the reference only after rating the source archive', async () => {
const agent = {
id: 'agent-source-owned',
status: 'completed',
completedAt: '2026-08-06T10:00:00.000Z',
metadata: { taskType: 'user', taskDescription: 'Fix an example regression' },
};
const dir = join(mockCosState.agentsDir, '2026-08-06', agent.id);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'metadata.json'), JSON.stringify(agent));
mockCosState.state = { agents: {} };
mockAgentIndex.entries.set(agent.id, '2026-08-06');
await getPendingAgentFeedback();
await submitAgentFeedback(agent.id, { rating: 'positive' });
await expect(listPendingAgentFeedbackRefs()).resolves.toEqual([]);
await expect(getPendingAgentFeedbackCount()).resolves.toBe(0);
});
it('aggregates archived feedback and de-duplicates a live copy of the same agent', async () => {
const dateBucket = '2026-08-01';
const archived = [
{
id: 'agent-archived',
metadata: { taskDescription: 'Fix the example bug' },
feedback: { rating: 'negative', comment: 'The regression remained.', submittedAt: '2026-08-01T11:00:00.000Z' }
},
{
id: 'agent-duplicate',
metadata: { taskDescription: 'Write example tests' },
feedback: { rating: 'positive', submittedAt: '2026-08-01T10:00:00.000Z' }
}
];
for (const agent of archived) {
const dir = join(mockCosState.agentsDir, dateBucket, agent.id);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'metadata.json'), JSON.stringify(agent));
mockAgentIndex.entries.set(agent.id, dateBucket);
}
mockCosState.state.agents = {
'agent-duplicate': {
...archived[1],
feedback: { rating: 'positive', submittedAt: '2026-08-01T12:00:00.000Z' }
},
'agent-live': {
id: 'agent-live',
metadata: { taskDescription: 'Document the example workflow' },
feedback: { rating: 'neutral', submittedAt: '2026-08-01T13:00:00.000Z' }
}
};
await expect(getFeedbackStats()).resolves.toEqual({
total: 3,
positive: 1,
negative: 1,
neutral: 1,
satisfactionRate: 33,
byTaskType: {
'auto-fix': { positive: 0, negative: 1, neutral: 0, total: 1 },
'external/untyped': { positive: 1, negative: 0, neutral: 1, total: 2 }
},
recentWithComments: [{
agentId: 'agent-archived',
taskDescription: 'Fix the example bug',
rating: 'negative',
comment: 'The regression remained.',
submittedAt: '2026-08-01T11:00:00.000Z'
}]
});
});
it('classifies claim-issue and self-improvement agents with parity between feedback and learning (#10403)', async () => {
// Issue: feedback stats used a divergent extractTaskType taxonomy. Verify
// that feedback classification now matches the canonical learning one by
// testing claim-issue (user task / issue description) and self-improvement
// agents get the same bucket in both paths.
const { extractTaskType: learningExtractTaskType } = await import('./taskLearning/store.js');
// Claim-issue agent: mimics a /do:next claim task with issue metadata
const claimAgent = {
id: 'agent-claim-issue',
taskId: 'github-issue-999',
status: 'completed',
completedAt: '2026-08-02T10:00:00.000Z',
metadata: {
taskType: 'user',
taskDescription: 'Investigate and fix parsing regression',
},
feedback: { rating: 'positive', submittedAt: '2026-08-02T11:00:00.000Z' }
};
// Self-improvement agent: metadata includes analysisType
const selfImprovementAgent = {
id: 'agent-self-improve',
status: 'completed',
completedAt: '2026-08-02T12:00:00.000Z',
metadata: {
taskType: 'internal',
analysisType: 'code-review',
taskDescription: 'Review recent PRs for quality',
},
feedback: { rating: 'positive', submittedAt: '2026-08-02T13:00:00.000Z' }
};
mockCosState.state.agents = {
[claimAgent.id]: claimAgent,
[selfImprovementAgent.id]: selfImprovementAgent,
};
// Get feedback classification
const feedbackStats = await getFeedbackStats();
// Get learning classification (what taskLearning/store.js would use)
const claimLearningType = learningExtractTaskType({
description: claimAgent.metadata.taskDescription,
metadata: claimAgent.metadata,
taskType: claimAgent.metadata.taskType,
});
const selfImproveLearningType = learningExtractTaskType({
description: selfImprovementAgent.metadata.taskDescription,
metadata: selfImprovementAgent.metadata,
taskType: selfImprovementAgent.metadata.taskType,
});
// Verify feedback and learning classifications match
expect(Object.keys(feedbackStats.byTaskType)).toContain(claimLearningType);
expect(Object.keys(feedbackStats.byTaskType)).toContain(selfImproveLearningType);
expect(feedbackStats.byTaskType[claimLearningType]).toEqual({
positive: 1, negative: 0, neutral: 0, total: 1
});
expect(feedbackStats.byTaskType[selfImproveLearningType]).toEqual({
positive: 1, negative: 0, neutral: 0, total: 1
});
});
});
// ── Operator-action ledger (#5594) ──────────────────────────────────────────
//
// The hook lives in `submitAgentFeedback` rather than the HTTP route so every
// caller records the rating, and it runs AFTER the CoS state lock releases so a
// ledger write never holds that lock across I/O.
describe('submitAgentFeedback records the rating (#5594)', () => {
beforeEach(async () => {
mockCosState.state = {
agents: {
'agent-1': {
id: 'agent-1',
taskId: 'task-42',
status: 'completed',
completedAt: '2026-08-01T10:00:00.000Z',
metadata: { taskType: 'user', taskDescription: 'Fix the failing render test' },
},
},
};
mockAgentIndex.entries = new Map();
resetPendingAgentFeedbackStore();
await rm(join(ledgerRoot, 'user-action-events.json'), { force: true });
});
it('writes a cos.agent.feedback row with the rating, comment, and derived task type', async () => {
const result = await submitAgentFeedback('agent-1', { rating: 'negative', comment: 'Missed the root cause' });
expect(result).toMatchObject({ success: true });
// The internal feedbackData carrier must not leak into the caller's response.
expect(result).not.toHaveProperty('feedbackData');
const [event] = await listUserActions({ type: 'cos.agent.feedback' });
expect(event).toMatchObject({
actor: 'user',
target: 'agent-1',
targetName: 'Fix the failing render test',
source: { service: 'cosAgentFeedback', fn: 'submitAgentFeedback' },
});
expect(event.payload).toMatchObject({
agentId: 'agent-1',
taskId: 'task-42',
rating: 'negative',
comment: 'Missed the root cause',
// extractTaskType uses the canonical classifier from taskLearning/store.js,
// which prioritizes taskType field and metadata patterns over description.
taskType: 'user-task',
});
expect(event.dedupeKey).toBe(`cos.agent.feedback:agent-1:${event.happenedAt}`);
});
it('records nothing when the agent is not in a ratable state', async () => {
mockCosState.state.agents['agent-1'].status = 'running';
await expect(submitAgentFeedback('agent-1', { rating: 'positive' })).rejects.toThrow(/completed agents/);
expect(await listUserActions({ type: 'cos.agent.feedback' })).toEqual([]);
});
});
/**
* The scalar badge and the paged list answer from the completion-order
* eligibility projection (#7968). Each case here leaves the archive record OFF
* disk: a reader that still scans pending metadata to decide eligibility cannot
* produce these answers.
*/
describe('pending feedback served from the eligibility projection', () => {
const orderFile = () => join(mockCosState.agentsDir, 'index.order.json');
const seedProjection = async (entries) => {
await mkdir(mockCosState.agentsDir, { recursive: true });
await writeFile(orderFile(), JSON.stringify(encodeCompletionOrder(new Map(entries))));
resetCompletionOrderIndex();
};
// Enroll a durable reference the way a live completion does, then evict the
// live record so only the archive side remains.
const enrollThenEvict = async (agents) => {
mockCosState.state.agents = Object.fromEntries(agents.map((agent) => [agent.id, agent]));
await getPendingAgentFeedback();
mockCosState.state = { agents: {} };
for (const agent of agents) mockAgentIndex.entries.set(agent.id, agent.completedAt.slice(0, 10));
};
const pendingAgent = (id, completedAt) => ({
id, status: 'completed', completedAt, metadata: { taskType: 'user', taskDescription: `Example run ${id}` },
});
beforeEach(async () => {
mockCosState.state = { agents: {} };
mockAgentIndex.entries = new Map();
resetPendingAgentFeedbackStore();
resetCompletionOrderIndex();
await rm(join(ledgerRoot, 'cos-pending-agent-feedback.json'), { force: true });
await rm(orderFile(), { force: true });
});
afterAll(async () => {
await rm(mockCosState.agentsDir, { recursive: true, force: true });
});
it('counts an eligible archived run without reading its metadata', async () => {
const agent = pendingAgent('agent-projected-open', '2026-08-10T10:00:00.000Z');
await enrollThenEvict([agent]);
await seedProjection([[agent.id, { completedAt: agent.completedAt, completed: true, feedbackEligible: true }]]);
await expect(getPendingAgentFeedbackCount()).resolves.toBe(1);
});
it('publishes feedback only after the pending count reflects the saved rating', async () => {
const agent = pendingAgent('agent-feedback-event', '2026-08-10T10:00:00.000Z');
await enrollThenEvict([agent]);
await seedProjection([[agent.id, { completedAt: agent.completedAt, completed: true, feedbackEligible: true }]]);
const dir = join(mockCosState.agentsDir, agent.completedAt.slice(0, 10), agent.id);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'metadata.json'), JSON.stringify(agent));
await expect(getPendingAgentFeedbackCount()).resolves.toBe(1);
let countAtEvent;
const onFeedback = vi.fn(() => { countAtEvent = getPendingAgentFeedbackCount(); });
cosEvents.on('agent:feedback', onFeedback);
await submitAgentFeedback(agent.id, { rating: 'positive' }).finally(() => {
cosEvents.off('agent:feedback', onFeedback);
});
expect(onFeedback).toHaveBeenCalledExactlyOnceWith({
agentId: agent.id,
feedback: { rating: 'positive', comment: null, submittedAt: expect.any(String) },
});
await expect(countAtEvent).resolves.toBe(0);
});
it('stops counting a run the projection records as already rated', async () => {
const agent = pendingAgent('agent-projected-rated', '2026-08-11T10:00:00.000Z');
await enrollThenEvict([agent]);
await seedProjection([[agent.id, { completedAt: agent.completedAt, completed: true, feedbackEligible: false }]]);
await expect(getPendingAgentFeedbackCount()).resolves.toBe(0);
});
it('pages newest id first and hydrates only the rows it returns', async () => {
const agents = ['agent-page-a', 'agent-page-b', 'agent-page-c']
.map((id, i) => pendingAgent(id, `2026-08-12T1${i}:00:00.000Z`));
await enrollThenEvict(agents);
await seedProjection(agents.map((agent) => [agent.id,
{ completedAt: agent.completedAt, completed: true, feedbackEligible: true }]));
// Only the row the page returns exists on disk — `total` still sees all three,
// which a reader that hydrated the whole list up front could not report.
const newest = agents.at(-1);
const dir = join(mockCosState.agentsDir, newest.completedAt.slice(0, 10), newest.id);
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'metadata.json'), JSON.stringify(newest));
await expect(getPendingAgentFeedbackPage({ limit: 1 })).resolves.toMatchObject({
items: [newest],
total: 3,
nextCursor: newest.id,
});
});
});
// ledgerRoot is the module-scope mkdtemp root every describe block above
// shares. Remove it exactly once, after every test in the file has run —
// nested inside one describe block, its own afterAll would fire before a
// LATER describe block finishes writing to it (#9032).
afterAll(async () => {
await rm(ledgerRoot, { recursive: true, force: true });
});