Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop goal-fidelity holding pr-watcher runs whose only diff is a test CI repair

[._.]/ Adam Eivy7d269abda5e1
diff --git a/server/lib/goalFidelity.js b/server/lib/goalFidelity.js
index e9de946112..457a41039c 100644
--- a/server/lib/goalFidelity.js
+++ b/server/lib/goalFidelity.js
@@ -322,6 +322,27 @@ export function mergeOutcomeReview({ number, prState }) {
};
}
+/**
+ * Whether a pr-watcher run's diff is only the incidental CI repair of a
+ * forge-side remediation, which a diff cannot judge.
+ *
+ * pr-watcher's objective is forge state (wait for CI, merge N listed PRs). The
+ * work is GitHub operations that leave no repo diff; the only trace a run
+ * leaves is a stray fix that unblocked CI, and the reviewer then judges that
+ * lone commit against "remediate 10 PRs" and answers `rethink`. That held a run
+ * whose PRs all merged.
+ *
+ * Deliberately narrow: only the pr-watcher type, and only a diff whose every
+ * file is a test file. A pr-watcher customised into a feature-committing flow
+ * touches production code, so it keeps the ordinary review and its hold.
+ */
+export function isForgeRemediationTestFixDiff({ analysisType, diff }) {
+ if (analysisType !== 'pr-watcher' || typeof diff !== 'string') return false;
+ const files = [...diff.matchAll(/^diff --git a\/(\S+) b\/\S+/gm)].map(m => m[1]);
+ return files.length > 0
+ && files.every(f => /(^|\/)(__tests__|tests?)\//.test(f) || /\.(test|spec)\.[cm]?[jt]sx?$/.test(f));
+}
+
/**
* Production witnesses for a removal that may be incomplete. This is evidence,
* never a verdict: compatibility paths, moves and comments can legitimately
diff --git a/server/services/agentFinalization.goalFidelity.test.js b/server/services/agentFinalization.goalFidelity.test.js
index f49c243852..4f381dbc65 100644
--- a/server/services/agentFinalization.goalFidelity.test.js
+++ b/server/services/agentFinalization.goalFidelity.test.js
@@ -253,6 +253,41 @@ describe('finalizeAgent — goal-fidelity gate', () => {
expect(completion()).toMatchObject({ success: true });
});
+ describe('forge-side PR remediation whose only diff is a test CI repair', () => {
+ const task = {
+ id: 'pr-watcher-remediation', taskType: 'internal',
+ description: 'Wait for CI and merge the listed trusted pull requests: #10727, #10725, #10722.',
+ metadata: { analysisType: 'pr-watcher' },
+ };
+ const diffOf = file => `diff --git a/${file} b/${file}\n--- a/${file}\n+++ b/${file}\n@@ -1 +1 @@\n-budget(1000)\n+budget(5000)\n`;
+ beforeEach(() => {
+ runLocalGoalFidelityReviewMock.mockResolvedValue(verdict({
+ verdict: 'rethink', missing: ['Evidence the PRs were remediated'], unrequested: [], evidence: 'Only a timing budget changed.',
+ }));
+ });
+
+ it('declines a diff-only judgement when the run window holds only a test-file CI repair', async () => {
+ runWindowDiffMock.mockResolvedValue({ diff: diffOf('server/lib/toon.test.js'), base: 'abc', truncated: false, reason: null });
+ await finalize({ task, outputBuffer: 'Merged 10 PRs.' });
+ expect(runLocalGoalFidelityReviewMock).not.toHaveBeenCalled();
+ expect(completion()).toMatchObject({ success: true });
+ expect(completion().goalFidelity).toBeUndefined();
+ });
+
+ it('still judges a pr-watcher run that changed production code', async () => {
+ runWindowDiffMock.mockResolvedValue({ diff: diffOf('server/lib/toon.js'), base: 'abc', truncated: false, reason: null });
+ await finalize({ task, outputBuffer: 'Merged 10 PRs.' });
+ expect(runLocalGoalFidelityReviewMock).toHaveBeenCalled();
+ expect(completion()).toMatchObject({ success: false, completionReason: GOAL_FIDELITY_CATEGORY });
+ });
+
+ it('still judges a test-only diff from a non-pr-watcher task', async () => {
+ runWindowDiffMock.mockResolvedValue({ diff: diffOf('server/lib/toon.test.js'), base: 'abc', truncated: false, reason: null });
+ await finalize({ task: { ...task, metadata: {} }, outputBuffer: 'done' });
+ expect(runLocalGoalFidelityReviewMock).toHaveBeenCalled();
+ });
+ });
+
describe('dependency audit summary with work outside the diff (#7899)', () => {
// Synthetic reconstruction: the deliverable is a completed inventory and
// audit, with no warranted bump. The only committed change is its note.
diff --git a/server/services/agentFinalization.js b/server/services/agentFinalization.js
index 995b23ff15..12f8d300cf 100644
--- a/server/services/agentFinalization.js
+++ b/server/services/agentFinalization.js
@@ -51,6 +51,7 @@ import {
goalFidelityLogMarker,
isDependencyAuditSummaryDiff,
mergeOutcomeObjective,
+ isForgeRemediationTestFixDiff,
mergeOutcomeReview,
taskObjective,
} from '../lib/goalFidelity.js';
@@ -913,6 +914,10 @@ async function evaluateGoalFidelity({ task, workspacePath, startedAt }) {
return noFidelityVerdict('Dependency audit summary is not judgeable from a diff: independent forge queries found no open alerts or PRs, but the diff cannot verify the audit execution or scanner coverage.');
}
+ if (!claimFlow && isForgeRemediationTestFixDiff({ analysisType: resolveTaskHookType(task), diff })) {
+ return noFidelityVerdict('PR remediation is not judgeable from a diff: its objective is forge state (CI, merges), and the run window holds only a test-file CI repair.');
+ }
+
const result = await runLocalGoalFidelityReview({
backend: config.backend,
model: config.model,