Open-source SEO, GEO, and marketing skills for AI agents.
feat(goals): local shell metric source + supporting metrics with history
Yuting Zhongb3ca473dc723
diff --git a/notfair/src/app/(app)/[project]/goals/[agent]/page.tsx b/notfair/src/app/(app)/[project]/goals/[agent]/page.tsxindex 43f632ad..30212fde 100644--- a/notfair/src/app/(app)/[project]/goals/[agent]/page.tsx+++ b/notfair/src/app/(app)/[project]/goals/[agent]/page.tsx@@ -41,6 +41,12 @@ import { GoalChecksStrip } from "@/components/goal-checks-strip";import { GoalChecksList } from "@/components/goal-checks-list";import { RailSection } from "@/components/rail-section";import { listGoalPrs, type GoalPr } from "@/server/db/goal-prs";+import {+ listSupportMetrics,+ listSupportMetricSnapshots,+ type GoalSupportMetric,+} from "@/server/db/goal-support-metrics";+import { GoalSparkline } from "@/components/goal-sparkline";import { maybeSyncGoalPrs } from "@/server/goals/pr-sync";import { buildCheckSquares, currentStreak } from "@/lib/goal-streak";@@ -201,6 +207,7 @@ function GoalRail({// fresh state a moment later without blocking this render.// Merged/closed PRs leave the rail section; each stays reachable from// the check that created it (CheckRow.prs).+ const supportMetrics = listSupportMetrics(goal.id);const openPrs = listGoalPrs(goal.id, 100).filter((pr) => pr.state === "open");if (openPrs.length > 0) maybeSyncGoalPrs(goal.id);// First diary page for the lazy checks list; older pages stream in on@@ -355,6 +362,16 @@ function GoalRail({)}</RailCard>+ {supportMetrics.length > 0 && (+ <RailSection title="Supporting metrics" count={supportMetrics.length}>+ <ul className="m-0 flex list-none flex-col gap-3 p-0">+ {supportMetrics.map((m) => (+ <SupportMetricItem key={m.id} metric={m} />+ ))}+ </ul>+ </RailSection>+ )}+{(dueActions.length > 0 || gatedActions.length > 0) && (<RailSectiontitle="Open actions"@@ -465,6 +482,35 @@ function RailCard({ children }: { children: React.ReactNode }) {return <div className="ns-card p-3.5">{children}</div>;}+/** One supporting metric: current value, delta context, mini sparkline. */+function SupportMetricItem({ metric }: { metric: GoalSupportMetric }) {+ const values = listSupportMetricSnapshots(metric.id, 90).map((sn) => sn.value);+ return (+ <li className="text-[12px] leading-snug">+ <div className="flex items-baseline justify-between gap-2">+ <span className="text-[11px] text-[hsl(var(--notfair-ink-4))]">{metric.name}</span>+ <span className="font-medium tabular-nums">+ {formatMetric(metric.current_value)}+ <span className="ml-1.5 font-normal text-[11px] text-[hsl(var(--notfair-ink-4))]">+ baseline {formatMetric(metric.baseline_value)}+ </span>+ </span>+ </div>+ {values.length >= 2 && (+ <div className="mt-1">+ <GoalSparkline+ values={values}+ target={null}+ direction={metric.direction}+ width={330}+ height={36}+ />+ </div>+ )}+ </li>+ );+}+function RailStat({ k, v }: { k: string; v: string }) {return (<div>diff --git a/notfair/src/server/db/goal-support-metrics.test.ts b/notfair/src/server/db/goal-support-metrics.test.tsnew file mode 100644index 00000000..d10336e3--- /dev/null+++ b/notfair/src/server/db/goal-support-metrics.test.ts@@ -0,0 +1,195 @@+import { beforeAll, describe, expect, it, vi } from "vitest";++// Real better-sqlite3 against a tmpdir DB, per repo test conventions.+// MUST be hoisted: static imports evaluate before module-level statements,+// and db.ts captures NOTFAIR_DATA_DIR at import time — a plain assignment+// here would silently point the suite at the developer's live ~/.notfair.+vi.hoisted(() => {+ const { mkdtempSync } = require("node:fs") as typeof import("node:fs");+ const { tmpdir } = require("node:os") as typeof import("node:os");+ const { join } = require("node:path") as typeof import("node:path");+ process.env.NOTFAIR_DATA_DIR = mkdtempSync(join(tmpdir(), "notfair-support-"));+});++import { getDb } from "./db";+import { createGoal } from "./goals";+import {+ listSupportMetricSnapshots,+ listSupportMetrics,+ recordSupportMetricSnapshot,+ replaceSupportBackfillSnapshots,+ upsertSupportMetric,+} from "./goal-support-metrics";+import { handleAddSupportMetric } from "@/server/goals/handlers";++const SLUG = "proj";+let goalId: string;++beforeAll(() => {+ getDb()+ .prepare(+ "INSERT INTO projects (id, slug, display_name, created_at, harness_adapter) VALUES ('p1', ?, 'Proj', ?, 'codex-local')",+ )+ .run(SLUG, new Date().toISOString());+ goalId = createGoal({ project_slug: SLUG, agent_id: "agent-1", statement: "ship" }).id;+});++describe("goal-support-metrics db", () => {+ it("inserts, lists, and snapshots on upsert", () => {+ const m = upsertSupportMetric({+ goal_id: goalId,+ name: "PRs opened",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo 5"}`,+ direction: "increase",+ measured_value: 5,+ });+ expect(m.baseline_value).toBe(5);+ expect(m.current_value).toBe(5);+ expect(listSupportMetrics(goalId)).toHaveLength(1);+ const snaps = listSupportMetricSnapshots(m.id);+ expect(snaps).toHaveLength(1);+ expect(snaps[0]!.source).toBe("verify");+ });++ it("re-using the name redefines in place and keeps snapshot history", () => {+ const before = listSupportMetrics(goalId)[0]!;+ const m = upsertSupportMetric({+ goal_id: goalId,+ name: "PRs opened",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo 8"}`,+ measured_value: 8,+ });+ expect(m.id).toBe(before.id); // same row, not a duplicate+ expect(m.baseline_value).toBe(8);+ expect(m.direction).toBeNull(); // redefinition replaces the spec+ expect(listSupportMetrics(goalId)).toHaveLength(1);+ expect(listSupportMetricSnapshots(m.id).length).toBe(2);+ });++ it("tick snapshots advance current_value", () => {+ const m = listSupportMetrics(goalId)[0]!;+ recordSupportMetricSnapshot(m.id, 11, "tick");+ expect(listSupportMetrics(goalId)[0]!.current_value).toBe(11);+ });++ it("backfill replaces prior backfill, keeps live snapshots and current_value", () => {+ const m = listSupportMetrics(goalId)[0]!;+ const liveBefore = listSupportMetricSnapshots(m.id).filter(+ (s) => s.source !== "backfill",+ ).length;+ replaceSupportBackfillSnapshots(m.id, [+ { value: 1, created_at: "2026-07-01T00:00:00.000Z" },+ { value: 2, created_at: "2026-07-02T00:00:00.000Z" },+ ]);+ replaceSupportBackfillSnapshots(m.id, [+ { value: 3, created_at: "2026-07-03T00:00:00.000Z" },+ ]);+ const snaps = listSupportMetricSnapshots(m.id);+ expect(snaps.filter((s) => s.source === "backfill")).toHaveLength(1);+ expect(snaps.filter((s) => s.source !== "backfill")).toHaveLength(liveBefore);+ expect(listSupportMetrics(goalId)[0]!.current_value).toBe(11); // untouched+ });+});++describe("handleAddSupportMetric", () => {+ const ctx = { project_slug: SLUG, agent_id: "agent-1" };++ it("verifies through the local source and stores the metric", async () => {+ const r = await handleAddSupportMetric(+ {+ goal_id: goalId,+ name: "Open listing PRs (live)",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo 104"}`,+ direction: "increase",+ },+ ctx,+ );+ expect(r.ok).toBe(true);+ if (r.ok) expect(r.data.metric.current_value).toBe(104);+ });++ it("backfills history when history_args_json is given", async () => {+ const history = JSON.stringify({+ command: `echo '[{"date":"2026-07-01","value":100},{"date":"2026-07-02","value":102}]'`,+ });+ const r = await handleAddSupportMetric(+ {+ goal_id: goalId,+ name: "Open listing PRs (live)",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo 104"}`,+ history_args_json: history,+ },+ ctx,+ );+ expect(r.ok).toBe(true);+ if (r.ok) expect(r.data.backfilled).toBe(2);+ });++ it("fails with a fix-and-retry error when the history query is broken", async () => {+ const r = await handleAddSupportMetric(+ {+ goal_id: goalId,+ name: "Open listing PRs (live)",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo 104"}`,+ history_args_json: `{"command":"echo not-an-array"}`,+ },+ ctx,+ );+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("history query failed");+ });++ it("rejects a failing source with the measurement error", async () => {+ const r = await handleAddSupportMetric(+ {+ goal_id: goalId,+ name: "Broken",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo nope"}`,+ },+ ctx,+ );+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("verification failed");+ });++ it("rejects an unconnected MCP source key", async () => {+ const r = await handleAddSupportMetric(+ {+ goal_id: goalId,+ name: "Clicks",+ source_key: "notfair-googlesearchconsole",+ source_tool: "runScript",+ source_args_json: `{}`,+ },+ ctx,+ );+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("No connected MCP");+ });++ it("rejects another agent's goal", async () => {+ const r = await handleAddSupportMetric(+ {+ goal_id: goalId,+ name: "X",+ source_key: "local",+ source_tool: "shell",+ source_args_json: `{"command":"echo 1"}`,+ },+ { project_slug: SLUG, agent_id: "agent-2" },+ );+ expect(r.ok).toBe(false);+ });+});diff --git a/notfair/src/server/db/goal-support-metrics.ts b/notfair/src/server/db/goal-support-metrics.tsnew file mode 100644index 00000000..cdc381dd--- /dev/null+++ b/notfair/src/server/db/goal-support-metrics.ts@@ -0,0 +1,163 @@+import { randomUUID } from "node:crypto";+import { getDb } from "./db";+import type { MetricDirection } from "./goals";++/**+ * Supporting metrics — extra numbers the loop measures alongside the+ * goal's primary metric. A goal is judged on ONE number (target,+ * deadline, achieved/failed); supporting metrics are the leading+ * indicators and diagnostics around it ("PRs opened" while merges are+ * the target, "impressions" while clicks are the target). Same trust+ * rule as the primary: the agent authors the source once, the platform+ * measures it on every check.+ */++export type GoalSupportMetric = {+ id: string;+ goal_id: string;+ name: string;+ source_key: string;+ source_tool: string;+ source_args_json: string;+ direction: MetricDirection | null;+ baseline_value: number;+ current_value: number;+ created_at: string;+ updated_at: string;+};++export type GoalSupportMetricSnapshot = {+ id: string;+ metric_id: string;+ value: number;+ source: "verify" | "tick" | "backfill";+ created_at: string;+};++function now(): string {+ return new Date().toISOString();+}++export type UpsertSupportMetricInput = {+ goal_id: string;+ name: string;+ source_key: string;+ source_tool: string;+ source_args_json: string;+ direction?: MetricDirection | null;+ /** The value the platform just measured while verifying the source. */+ measured_value: number;+};++/**+ * Add a supporting metric, or redefine it by name (fixing a wrong query+ * shouldn't need a second tool). Redefinition resets baseline + current+ * to the freshly measured value; snapshot history is kept — the chart+ * shows the metric's life, not just its latest definition.+ */+export function upsertSupportMetric(input: UpsertSupportMetricInput): GoalSupportMetric {+ const db = getDb();+ const ts = now();+ db.prepare(+ `INSERT INTO goal_support_metrics+ (id, goal_id, name, source_key, source_tool, source_args_json,+ direction, baseline_value, current_value, created_at, updated_at)+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)+ ON CONFLICT(goal_id, name) DO UPDATE SET+ source_key = excluded.source_key,+ source_tool = excluded.source_tool,+ source_args_json = excluded.source_args_json,+ direction = excluded.direction,+ baseline_value = excluded.baseline_value,+ current_value = excluded.current_value,+ updated_at = excluded.updated_at`,+ ).run(+ randomUUID(),+ input.goal_id,+ input.name,+ input.source_key,+ input.source_tool,+ input.source_args_json,+ input.direction ?? null,+ input.measured_value,+ input.measured_value,+ ts,+ ts,+ );+ const metric = getSupportMetricByName(input.goal_id, input.name)!;+ recordSupportMetricSnapshot(metric.id, input.measured_value, "verify");+ return metric;+}++export function getSupportMetricByName(+ goal_id: string,+ name: string,+): GoalSupportMetric | null {+ const row = getDb()+ .prepare("SELECT * FROM goal_support_metrics WHERE goal_id = ? AND name = ?")+ .get(goal_id, name);+ return (row as GoalSupportMetric) ?? null;+}++export function listSupportMetrics(goal_id: string): GoalSupportMetric[] {+ return getDb()+ .prepare(+ "SELECT * FROM goal_support_metrics WHERE goal_id = ? ORDER BY created_at ASC, rowid ASC",+ )+ .all(goal_id) as GoalSupportMetric[];+}++export function recordSupportMetricSnapshot(+ metric_id: string,+ value: number,+ source: GoalSupportMetricSnapshot["source"] = "tick",+): GoalSupportMetricSnapshot {+ const db = getDb();+ const id = randomUUID();+ const ts = now();+ db.prepare(+ "INSERT INTO goal_support_metric_snapshots (id, metric_id, value, source, created_at) VALUES (?, ?, ?, ?, ?)",+ ).run(id, metric_id, value, source, ts);+ db.prepare(+ "UPDATE goal_support_metrics SET current_value = ?, updated_at = ? WHERE id = ?",+ ).run(value, ts, metric_id);+ return { id, metric_id, value, source, created_at: ts };+}++/**+ * Replace the metric's backfilled history with fresh backdated points —+ * same contract as the primary's replaceBackfillSnapshots: live+ * verify/tick snapshots are untouched, and current_value is not updated.+ */+export function replaceSupportBackfillSnapshots(+ metric_id: string,+ points: Array<{ value: number; created_at: string }>,+): number {+ const db = getDb();+ const insert = db.prepare(+ "INSERT INTO goal_support_metric_snapshots (id, metric_id, value, source, created_at) VALUES (?, ?, ?, 'backfill', ?)",+ );+ const tx = db.transaction(() => {+ db.prepare(+ "DELETE FROM goal_support_metric_snapshots WHERE metric_id = ? AND source = 'backfill'",+ ).run(metric_id);+ for (const p of points) insert.run(randomUUID(), metric_id, p.value, p.created_at);+ });+ tx();+ return points.length;+}++export function listSupportMetricSnapshots(+ metric_id: string,+ limit = 90,+): GoalSupportMetricSnapshot[] {+ // Ascending so sparklines read left→right; LIMIT applies to the tail.+ return getDb()+ .prepare(+ `SELECT id, metric_id, value, source, created_at FROM (+ SELECT *, rowid AS _rid FROM goal_support_metric_snapshots+ WHERE metric_id = ? ORDER BY created_at DESC, rowid DESC LIMIT ?+ ) ORDER BY created_at ASC, _rid ASC`,+ )+ .all(metric_id, limit) as GoalSupportMetricSnapshot[];+}diff --git a/notfair/src/server/db/schema.ts b/notfair/src/server/db/schema.tsindex 8cfafc0d..f7b0980d 100644--- a/notfair/src/server/db/schema.ts+++ b/notfair/src/server/db/schema.ts@@ -147,6 +147,36 @@ CREATE TABLE IF NOT EXISTS goal_metric_snapshots (CREATE INDEX IF NOT EXISTS idx_goal_snapshots_goalON goal_metric_snapshots(goal_id, created_at);+-- Supporting metrics: extra numbers the loop measures alongside the goal's+-- primary metric (leading indicators, diagnostics). No target semantics —+-- the goal is still judged on the ONE primary metric. Self-contained tables+-- (not columns on goals/goal_metric_snapshots) because schema boot is+-- CREATE-IF-NOT-EXISTS only: existing databases never gain columns.+CREATE TABLE IF NOT EXISTS goal_support_metrics (+ id TEXT PRIMARY KEY,+ goal_id TEXT NOT NULL REFERENCES goals(id) ON DELETE CASCADE,+ name TEXT NOT NULL,+ source_key TEXT NOT NULL,+ source_tool TEXT NOT NULL,+ source_args_json TEXT NOT NULL,+ direction TEXT CHECK (direction IN ('increase','decrease')),+ baseline_value REAL NOT NULL,+ current_value REAL NOT NULL,+ created_at TEXT NOT NULL,+ updated_at TEXT NOT NULL,+ UNIQUE(goal_id, name)+);++CREATE TABLE IF NOT EXISTS goal_support_metric_snapshots (+ id TEXT PRIMARY KEY,+ metric_id TEXT NOT NULL REFERENCES goal_support_metrics(id) ON DELETE CASCADE,+ value REAL NOT NULL,+ source TEXT NOT NULL DEFAULT 'tick' CHECK (source IN ('verify','tick','backfill')),+ created_at TEXT NOT NULL+);+CREATE INDEX IF NOT EXISTS idx_goal_support_snapshots_metric+ ON goal_support_metric_snapshots(metric_id, created_at);+CREATE TABLE IF NOT EXISTS goal_learnings (id TEXT PRIMARY KEY,goal_id TEXT NOT NULL REFERENCES goals(id) ON DELETE CASCADE,diff --git a/notfair/src/server/goals/handlers.ts b/notfair/src/server/goals/handlers.tsindex 99c26504..4ac21e53 100644--- a/notfair/src/server/goals/handlers.ts+++ b/notfair/src/server/goals/handlers.ts@@ -27,6 +27,12 @@ import {type MetricDirection,} from "@/server/db/goals";import { getProject } from "@/server/db/projects";+import {+ listSupportMetrics,+ replaceSupportBackfillSnapshots,+ upsertSupportMetric,+ type GoalSupportMetric,+} from "@/server/db/goal-support-metrics";import { findMcpToken } from "@/server/mcp/tokens";import {PROJECT_BRIEF_MAX_BYTES,@@ -34,7 +40,7 @@ import {writeProjectBrief,} from "@/server/onboarding/project-brief";import type { Project } from "@/types";-import { runHistorySource, runMetricSource } from "./metric";+import { LOCAL_SOURCE_KEY, runHistorySource, runMetricSource } from "./metric";/*** Handlers for NotFair's MCP tools — the goal agent's coordination@@ -84,6 +90,10 @@ export type GoalStateView = {recent_learnings: GoalLearning[];recent_ticks: Array<Pick<GoalTick, "tick_number" | "status" | "metric_value" | "summary" | "started_at">>;metric_history: Array<{ value: number; created_at: string }>;+ /** Supporting metrics: measured every check, context only, never judged. */+ supporting_metrics: Array<+ Pick<GoalSupportMetric, "name" | "direction" | "baseline_value" | "current_value">+ >;/** Sum of spend_usd across your logged (non-abandoned) actions. */logged_spend_usd: number;/** Other agents' still-gated resources in this workspace — equally untouchable. */@@ -119,6 +129,12 @@ export async function handleGetGoal(value: s.value,created_at: s.created_at,})),+ supporting_metrics: listSupportMetrics(goal.id).map((m) => ({+ name: m.name,+ direction: m.direction,+ baseline_value: m.baseline_value,+ current_value: m.current_value,+ })),logged_spend_usd: loggedSpendTotal(goal.id),gated_by_other_agents: listGatedActionsForOtherAgents(ctx.project_slug, goal.id).map((a) => ({@@ -205,10 +221,13 @@ export async function handleProposeGoalMetric(error: "Define the goal first (define_goal) — a metric needs an ambition to measure.",};}- if (!findMcpToken(ctx.project_slug, input.metric_source_key)) {+ if (+ input.metric_source_key !== LOCAL_SOURCE_KEY &&+ !findMcpToken(ctx.project_slug, input.metric_source_key)+ ) {return {ok: false,- error: `No connected MCP '${input.metric_source_key}' for this project. Use a connected data source.`,+ error: `No connected MCP '${input.metric_source_key}' for this project. Use a connected data source, or the '${LOCAL_SOURCE_KEY}' shell source.`,};}@@ -266,7 +285,10 @@ export async function handleBackfillHistory(const r = resolveGoal(input.goal_id, ctx, { requireOwner: true });if (!r.ok) return r;const goal = r.data;- if (!findMcpToken(ctx.project_slug, input.source_key)) {+ if (+ input.source_key !== LOCAL_SOURCE_KEY &&+ !findMcpToken(ctx.project_slug, input.source_key)+ ) {return { ok: false, error: `No connected MCP '${input.source_key}' for this project.` };}const run = await runHistorySource(ctx.project_slug, {@@ -296,6 +318,102 @@ export async function handleBackfillHistory(};}+// ── add_supporting_metric ────────────────────────────────────────────────++export type AddSupportMetricInput = {+ goal_id: string;+ name: string;+ source_key: string;+ source_tool: string;+ source_args_json: string;+ direction?: MetricDirection;+ /** Optional date-segmented history query (same source) so the metric's+ * sparkline has a past from day one — mirrors backfill_metric_history. */+ history_args_json?: string;+};++/**+ * Attach a supporting metric to the goal — measured by the platform on+ * every check alongside the primary metric, shown on the Goal tab, and+ * carried in every tick brief. No target semantics: the goal is still+ * judged on its ONE primary metric. Same verification gate as the+ * primary (the platform re-runs the exact call), and calling again with+ * the same name redefines the metric in place.+ */+export async function handleAddSupportMetric(+ input: AddSupportMetricInput,+ ctx: HandlerContext,+): Promise<HandlerResult<{ metric: GoalSupportMetric; backfilled: number }>> {+ const r = resolveGoal(input.goal_id, ctx, { requireOwner: true });+ if (!r.ok) return r;+ const goal = r.data;++ const name = input.name.trim();+ if (!name) return { ok: false, error: "Give the supporting metric a name." };+ if (goal.metric_name && name === goal.metric_name) {+ return {+ ok: false,+ error: `'${name}' is the goal's primary metric — supporting metrics measure something else.`,+ };+ }+ if (+ input.source_key !== LOCAL_SOURCE_KEY &&+ !findMcpToken(ctx.project_slug, input.source_key)+ ) {+ return {+ ok: false,+ error: `No connected MCP '${input.source_key}' for this project. Use a connected data source, or the '${LOCAL_SOURCE_KEY}' shell source.`,+ };+ }++ const measured = await runMetricSource(ctx.project_slug, {+ key: input.source_key,+ tool: input.source_tool,+ args_json: input.source_args_json,+ });+ if (!measured.ok) {+ return {+ ok: false,+ error: `Server-side verification failed — fix the query and add again. ${measured.error}`,+ };+ }++ const metric = upsertSupportMetric({+ goal_id: goal.id,+ name,+ source_key: input.source_key,+ source_tool: input.source_tool,+ source_args_json: input.source_args_json,+ direction: input.direction ?? null,+ measured_value: measured.value,+ });++ // Optional history, verified and executed the same way as the metric.+ // A broken history query fails the call AFTER the metric is stored —+ // the metric itself verified fine; the agent fixes and re-adds.+ let backfilled = 0;+ if (input.history_args_json) {+ const run = await runHistorySource(ctx.project_slug, {+ key: input.source_key,+ tool: input.source_tool,+ args_json: input.history_args_json,+ });+ if (!run.ok) {+ return {+ ok: false,+ error: `Metric stored, but the history query failed — fix it and call again (same name redefines). ${run.error}`,+ };+ }+ const nowIso = new Date().toISOString();+ const points = run.points.filter((p) => p.date <= nowIso);+ backfilled = replaceSupportBackfillSnapshots(+ metric.id,+ points.map((p) => ({ value: p.value, created_at: p.date })),+ );+ }+ return { ok: true, data: { metric, backfilled } };+}+// ── propose_target (chat intake, step 3) ─────────────────────────────────function targetDirectionError(diff --git a/notfair/src/server/goals/metric.test.ts b/notfair/src/server/goals/metric.test.tsnew file mode 100644index 00000000..49b58095--- /dev/null+++ b/notfair/src/server/goals/metric.test.ts@@ -0,0 +1,102 @@+import { describe, expect, it, vi } from "vitest";++// Real better-sqlite3 against a tmpdir DB, per repo test conventions.+// MUST be hoisted: static imports evaluate before module-level statements,+// and db.ts captures NOTFAIR_DATA_DIR at import time — a plain assignment+// here would silently point the suite at the developer's live ~/.notfair.+vi.hoisted(() => {+ const { mkdtempSync } = require("node:fs") as typeof import("node:fs");+ const { tmpdir } = require("node:os") as typeof import("node:os");+ const { join } = require("node:path") as typeof import("node:path");+ process.env.NOTFAIR_DATA_DIR = mkdtempSync(join(tmpdir(), "notfair-metric-"));+});++import {+ LOCAL_SOURCE_KEY,+ LOCAL_SOURCE_TOOL,+ parseMetricValue,+ runHistorySource,+ runMetricSource,+} from "./metric";++const SLUG = "proj";++function localSource(command: string, tool = LOCAL_SOURCE_TOOL) {+ return {+ key: LOCAL_SOURCE_KEY,+ tool,+ args_json: JSON.stringify({ command }),+ };+}++describe("parseMetricValue", () => {+ it("accepts bare numbers, numeric strings, and {value} wrappers", () => {+ expect(parseMetricValue(21)).toBe(21);+ expect(parseMetricValue("21.5")).toBe(21.5);+ expect(parseMetricValue({ value: 3 })).toBe(3);+ expect(parseMetricValue({ ok: true, result: { value: 9 } })).toBe(9);+ });++ it("rejects ambiguous payloads", () => {+ expect(parseMetricValue("")).toBeNull();+ expect(parseMetricValue([1, 2])).toBeNull();+ expect(parseMetricValue({ rows: 4 })).toBeNull();+ });+});++describe("runMetricSource (local shell)", () => {+ it("parses a bare number from stdout", async () => {+ const r = await runMetricSource(SLUG, localSource("echo 42"));+ expect(r).toEqual({ ok: true, value: 42 });+ });++ it("parses a {value} JSON payload from stdout", async () => {+ const r = await runMetricSource(SLUG, localSource(`echo '{"value": 7.5}'`));+ expect(r).toEqual({ ok: true, value: 7.5 });+ });++ it("rejects non-numeric output with the payload echoed back", async () => {+ const r = await runMetricSource(SLUG, localSource("echo not-a-number"));+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("not-a-number");+ });++ it("surfaces command failure", async () => {+ const r = await runMetricSource(SLUG, localSource("exit 3"));+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("Local command failed");+ });++ it("rejects a missing command argument", async () => {+ const r = await runMetricSource(SLUG, {+ key: LOCAL_SOURCE_KEY,+ tool: LOCAL_SOURCE_TOOL,+ args_json: "{}",+ });+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain('{"command"');+ });++ it("rejects tools other than 'shell' on the local source", async () => {+ const r = await runMetricSource(SLUG, localSource("echo 1", "runScript"));+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("'shell'");+ });+});++describe("runHistorySource (local shell)", () => {+ it("parses a {date, value} array from stdout, sorted ascending", async () => {+ const rows = `[{"date":"2026-07-02","value":2},{"date":"2026-07-01","value":1}]`;+ const r = await runHistorySource(SLUG, localSource(`echo '${rows}'`));+ expect(r.ok).toBe(true);+ if (r.ok) {+ expect(r.points.map((p) => p.value)).toEqual([1, 2]);+ }+ });++ it("rejects non-array output", async () => {+ const r = await runHistorySource(SLUG, localSource("echo 5"));+ expect(r.ok).toBe(false);+ if (!r.ok) expect(r.error).toContain("array of {date, value}");+ });+});diff --git a/notfair/src/server/goals/metric.ts b/notfair/src/server/goals/metric.tsindex b3b7b0ec..44219eea 100644--- a/notfair/src/server/goals/metric.ts+++ b/notfair/src/server/goals/metric.ts@@ -1,13 +1,21 @@+import { execFile } from "node:child_process";+import { existsSync } from "node:fs";+import { homedir } from "node:os";+import { promisify } from "node:util";import { mcpRpcAutoRefresh } from "@/server/mcp/rpc";import type { Goal } from "@/server/db/goals";+import { getProject } from "@/server/db/projects";++const execFileAsync = promisify(execFile);/*** Mechanical execution of a goal's stored metric query against a catalog- * MCP (e.g. notfair-googleads `runScript`). This is the loop's ground- * truth: the tick runner calls it BEFORE the agent wakes, so the agent- * never self-reports the number it is being judged on. The same path- * verifies the metric during intake (`propose_goal_metric`) — a metric- * that can't be measured here is rejected before the goal can activate.+ * MCP (e.g. notfair-googleads `runScript`) or the `local` shell source.+ * This is the loop's ground truth: the tick runner calls it BEFORE the+ * agent wakes, so the agent never self-reports the number it is being+ * judged on. The same path verifies the metric during intake+ * (`propose_goal_metric`) — a metric that can't be measured here is+ * rejected before the goal can activate.*/export type MetricMeasurement =@@ -31,6 +39,55 @@ export type MetricSource = {args_json: string;};+/**+ * The `local` source: instead of a catalog MCP, the platform runs a+ * shell command on this machine and parses its stdout as the value —+ * for ambitions no connected MCP can measure (GitHub PRs via `gh`, a+ * local SQLite ledger, a curl'd endpoint). The trust rule is unchanged:+ * the agent authors the command once, but every measurement is executed+ * by the platform itself, never self-reported. It also adds no new+ * capability — tick agents already run with full shell access here.+ *+ * Spec: key `local`, tool `shell`, args `{"command": "<sh command>"}`.+ * Runs from the project's codebase_path when it exists (use absolute+ * paths regardless), with the metric timeout applied.+ */+export const LOCAL_SOURCE_KEY = "local";+export const LOCAL_SOURCE_TOOL = "shell";++type LocalRun = { ok: true; text: string } | { ok: false; error: string };++async function runLocalShell(+ project_slug: string,+ args: Record<string, unknown>,+): Promise<LocalRun> {+ const command = args.command;+ if (typeof command !== "string" || !command.trim()) {+ return { ok: false, error: `Local source args must be {"command": "<shell command>"}` };+ }+ const codebase = getProject(project_slug)?.codebase_path;+ const cwd = codebase && existsSync(codebase) ? codebase : homedir();+ try {+ const { stdout } = await execFileAsync("/bin/sh", ["-c", command], {+ timeout: METRIC_TIMEOUT_MS,+ maxBuffer: 4 * 1024 * 1024,+ cwd,+ });+ return { ok: true, text: stdout.trim() };+ } catch (err) {+ const msg = err instanceof Error ? err.message : String(err);+ return { ok: false, error: `Local command failed: ${truncate(msg, 500)}` };+ }+}++/** Validate the (key, tool) pair for a local source. Null when not local. */+function localSourceError(source: MetricSource): string | null {+ if (source.key !== LOCAL_SOURCE_KEY) return null;+ return source.tool === LOCAL_SOURCE_TOOL+ ? null+ : `The '${LOCAL_SOURCE_KEY}' source only supports tool '${LOCAL_SOURCE_TOOL}'.`;+}+export function metricSourceFromGoal(goal: Goal): MetricSource | null {if (!goal.metric_source_key || !goal.metric_source_tool || !goal.metric_source_args_json) {return null;@@ -99,6 +156,27 @@ export async function runMetricSource(return { ok: false, error: "metric_source_args_json is not valid JSON" };}+ if (source.key === LOCAL_SOURCE_KEY) {+ const toolError = localSourceError(source);+ if (toolError) return { ok: false, error: toolError };+ const run = await runLocalShell(project_slug, args);+ if (!run.ok) return run;+ let payload: unknown = run.text;+ try {+ payload = JSON.parse(run.text);+ } catch {+ // Not JSON — parseMetricValue handles numeric strings.+ }+ const value = parseMetricValue(payload);+ if (value === null) {+ return {+ ok: false,+ error: `Local command must print a single number (or {value: number}) to stdout. Got: ${truncate(run.text, 300)}`,+ };+ }+ return { ok: true, value };+ }+const rpc = await mcpRpcAutoRefresh<ToolCallResult>(project_slug,source.key,@@ -204,6 +282,26 @@ export async function runHistorySource(} catch {return { ok: false, error: "history args_json is not valid JSON" };}+ if (source.key === LOCAL_SOURCE_KEY) {+ const toolError = localSourceError(source);+ if (toolError) return { ok: false, error: toolError };+ const run = await runLocalShell(project_slug, args);+ if (!run.ok) return run;+ let payload: unknown = run.text;+ try {+ payload = JSON.parse(run.text);+ } catch {+ // fall through — parseHistoryPoints rejects non-array payloads.+ }+ const points = parseHistoryPoints(payload);+ if (!points) {+ return {+ ok: false,+ error: `Local history command must print an array of {date, value} rows (1–400 points) to stdout. Got: ${truncate(run.text, 300)}`,+ };+ }+ return { ok: true, points };+ }const rpc = await mcpRpcAutoRefresh<ToolCallResult>(project_slug,source.key,diff --git a/notfair/src/server/goals/tick.ts b/notfair/src/server/goals/tick.tsindex 19c44b4c..9b083292 100644--- a/notfair/src/server/goals/tick.ts+++ b/notfair/src/server/goals/tick.ts@@ -33,7 +33,12 @@ import {} from "@/server/db/goals";import { listGoalPrs, type GoalPr } from "@/server/db/goal-prs";import { syncGoalPrs } from "./pr-sync";-import { measureGoalMetric, type MetricMeasurement } from "./metric";+import { measureGoalMetric, runMetricSource, type MetricMeasurement } from "./metric";+import {+ listSupportMetrics,+ recordSupportMetricSnapshot,+ type GoalSupportMetric,+} from "@/server/db/goal-support-metrics";/*** The goal loop's tick runner — one OODA iteration per invocation.@@ -62,6 +67,8 @@ export type TickContext = {tickNumber: number;nowIso: string;measurement: MetricMeasurement;+ /** Supporting metrics measured this check: value, or the error. */+ supportReadings: SupportReading[];targetMet: boolean | null;pastDeadline: boolean;actionsDueForReview: GoalAction[];@@ -80,6 +87,33 @@ function fmtValue(v: number | null): string {return v === null ? "—" : String(v);}+export type SupportReading = {+ metric: GoalSupportMetric;+ measurement: MetricMeasurement;+};++/**+ * Measure every supporting metric and record snapshots — same ground-truth+ * rule as the primary: the platform measures before the agent wakes. A+ * failing supporting metric never blocks the check; the error rides into+ * the brief for the agent to fix (redefine via add_supporting_metric).+ */+async function measureSupportMetrics(goal: Goal): Promise<SupportReading[]> {+ const readings: SupportReading[] = [];+ for (const metric of listSupportMetrics(goal.id)) {+ const measurement = await runMetricSource(goal.project_slug, {+ key: metric.source_key,+ tool: metric.source_tool,+ args_json: metric.source_args_json,+ });+ if (measurement.ok) {+ recordSupportMetricSnapshot(metric.id, measurement.value, "tick");+ }+ readings.push({ metric, measurement });+ }+ return readings;+}+/** One line of live PR state for the tick brief. */export function describePrForBrief(pr: GoalPr): string {const bits: string[] = [`[${pr.state.toUpperCase()}]`];@@ -138,6 +172,17 @@ export function buildTickMessage(ctx: TickContext): string {lines.push(`- Baseline: ${fmtValue(goal.baseline_value)} | Target: ${fmtValue(goal.target_value)} (${goal.metric_direction ?? "?"})`,);+ for (const r of ctx.supportReadings) {+ if (r.measurement.ok) {+ lines.push(+ `- [supporting] ${r.metric.name}: **${r.measurement.value}** (baseline ${r.metric.baseline_value}${r.metric.direction ? `, healthy = ${r.metric.direction}` : ""}) — context only, the goal is judged on the primary metric`,+ );+ } else {+ lines.push(+ `- [supporting] ${r.metric.name}: MEASUREMENT FAILED: ${r.measurement.error} — fix by redefining it via add_supporting_metric (same name)`,+ );+ }+ }lines.push("");lines.push("## Stop-condition flags");@@ -380,6 +425,9 @@ async function runClaimedTick(if (measurement.ok) {recordMetricSnapshot(goal.id, measurement.value, "tick");}+ // Supporting metrics too — snapshots land even on observe-only checks,+ // so their charts stay daily.+ const supportReadings = await measureSupportMetrics(goal);setGoalTickMetric(tick.id,measurement.ok ? measurement.value : null,@@ -419,6 +467,7 @@ async function runClaimedTick(tickNumber,nowIso,measurement,+ supportReadings,targetMet,pastDeadline,actionsDueForReview,diff --git a/notfair/src/server/mcp-server/tools.ts b/notfair/src/server/mcp-server/tools.tsindex 771b7930..79839657 100644--- a/notfair/src/server/mcp-server/tools.ts+++ b/notfair/src/server/mcp-server/tools.ts@@ -3,6 +3,7 @@ import { CronExpressionParser } from "cron-parser";import {handleAmendGoal,+ handleAddSupportMetric,handleBackfillHistory,handleDefineGoal,handleGetGoal,@@ -130,16 +131,18 @@ const proposeGoalMetricInput = z.object({metric_source_key: z.string().min(1)- .describe("Connected catalog MCP key, e.g. 'notfair-googleads'."),+ .describe(+ "Connected catalog MCP key (e.g. 'notfair-googleads'), or 'local' to measure via a shell command the platform runs on this machine.",+ ),metric_source_tool: z.string().min(1)- .describe("Tool to call on that server, usually 'runScript'."),+ .describe("Tool to call on that server, usually 'runScript'. For the 'local' source: 'shell'."),metric_source_args_json: z.string().min(1).describe(- "JSON-encoded arguments object for the tool call. The call must return a single number (or {value: number}). You must have TESTED this exact call yourself first.",+ "JSON-encoded arguments object for the tool call. The call must return a single number (or {value: number}). For 'local': {\"command\": \"<sh command>\"} whose stdout is that number. You must have TESTED this exact call yourself first.",),metric_direction: z.enum(["increase", "decrease"])@@ -164,13 +167,18 @@ const backfillHistoryInput = z.object({source_key: z.string().min(1)- .describe("Connected catalog MCP key (usually the same as the metric's)."),- source_tool: z.string().min(1).describe("Tool to call, usually 'runScript'."),+ .describe(+ "Connected catalog MCP key (usually the same as the metric's), or 'local' for a shell command.",+ ),+ source_tool: z+ .string()+ .min(1)+ .describe("Tool to call, usually 'runScript'. For the 'local' source: 'shell'."),source_args_json: z.string().min(1).describe(- "JSON-encoded arguments for a DATE-SEGMENTED query returning an array of {date: 'YYYY-MM-DD', value: number} — one point per day, ~30 days, same definition as the goal metric. TEST it yourself first.",+ "JSON-encoded arguments for a DATE-SEGMENTED query returning an array of {date: 'YYYY-MM-DD', value: number} — one point per day, ~30 days, same definition as the goal metric. For 'local': {\"command\": \"<sh command>\"} printing that array to stdout. TEST it yourself first.",),});@@ -185,6 +193,53 @@ async function handleBackfillHistoryTool(input: unknown): Promise<ToolResult> {);}+// ── add_supporting_metric ──────────────────────────────────────────────++const addSupportMetricInput = z.object({+ ...goalIdFields,+ name: z+ .string()+ .min(1)+ .describe("Human label incl. unit + window, e.g. 'Listing PRs open (live)'. Re-using a name redefines that metric."),+ source_key: z+ .string()+ .min(1)+ .describe(+ "Connected catalog MCP key (e.g. 'notfair-googleads'), or 'local' for a shell command.",+ ),+ source_tool: z+ .string()+ .min(1)+ .describe("Tool to call on that server, usually 'runScript'. For the 'local' source: 'shell'."),+ source_args_json: z+ .string()+ .min(1)+ .describe(+ "JSON-encoded arguments object for the tool call. Must return a single number (or {value: number}); for 'local': {\"command\": \"<sh command>\"} whose stdout is that number. TEST it yourself first.",+ ),+ direction: z+ .enum(["increase", "decrease"])+ .optional()+ .describe("Which way is healthy, if there is one — context only, never judged."),+ history_args_json: z+ .string()+ .optional()+ .describe(+ "Optional: DATE-SEGMENTED version of the same query (same source) returning an array of {date: 'YYYY-MM-DD', value: number}, ~30 days — gives the metric's chart a past from day one. Replaces any earlier backfill. TEST it yourself first.",+ ),+});++async function handleAddSupportMetricTool(input: unknown): Promise<ToolResult> {+ const parsed = addSupportMetricInput.safeParse(input);+ if (!parsed.success) return invalid(parsed.error);+ const { project_slug, agent_id, ...rest } = parsed.data;+ const r = await handleAddSupportMetric(rest, { project_slug, agent_id });+ if (!r.ok) return { ok: false, error: r.error };+ return txt(+ `Supporting metric '${r.data.metric.name}' verified — current value ${r.data.metric.current_value}${r.data.backfilled > 0 ? `, ${r.data.backfilled} history points backfilled` : ""}. The platform now measures it on every check; it appears on your Goal tab and in every tick brief. The goal is still judged on the primary metric only.`,+ );+}+// ── propose_target / amend_goal ────────────────────────────────────────const proposeTargetInput = z.object({@@ -506,6 +561,13 @@ export const TOOLS: ToolDefinition[] = [inputSchema: backfillHistoryInput,handler: handleBackfillHistoryTool,},+ {+ name: "add_supporting_metric",+ description:+ "Attach a SUPPORTING metric to your goal — a second number the platform measures on every check alongside the primary (leading indicators, diagnostics: 'PRs opened' while merges are the target). Platform-verified like the primary; re-using a name redefines that metric. No target semantics — the goal is judged on the primary metric only. Usable at any point in the goal's life.",+ inputSchema: addSupportMetricInput,+ handler: handleAddSupportMetricTool,+ },{name: "propose_target",description: