Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

test: retarget the agent-chat evals at review_resume selection

Khuram Niazafb2ceed5cbf
diff --git a/evals/agent-chat/assertions.ts b/evals/agent-chat/assertions.ts
index f1797ac..7dd5d2d 100644
--- a/evals/agent-chat/assertions.ts
+++ b/evals/agent-chat/assertions.ts
@@ -123,17 +123,16 @@ export function assertCallsGetResume(output: unknown): AssertionResult {
return { pass, score: pass ? 1 : 0, reason: pass ? 'called get_resume' : `expected one get_resume call, got: ${describe(calls)}` };
}
-// The three parse paths share this function, so passing here means the chat
-// transcript, the review sheet and the MCP handler all read it identically.
-export function assertReviewParses(output: unknown): AssertionResult {
- const { scores, body } = parseResumeReview(textOf(output));
- if (!scores) {
- return { pass: false, score: 0, reason: 'no parseable SCORES line in the review' };
- }
- const inRange = [scores.overall, scores.impact, scores.clarity, scores.atsCompatibility].every((n) => n >= 0 && n <= 100);
- const long = body.length >= 400;
- const pass = inRange && long;
- return { pass, score: pass ? 1 : 0, reason: pass ? `parsed overall=${scores.overall}, body ${body.length} chars` : `scores parsed but ${!inRange ? 'out of range' : `body only ${body.length} chars`}` };
+export function assertCallsReviewResume(output: unknown): AssertionResult {
+ const calls = parseToolCalls(output);
+ const pass = calls.length === 1 && calls[0].name === 'review_resume';
+ return {
+ pass,
+ score: pass ? 1 : 0,
+ reason: pass
+ ? 'called review_resume'
+ : `expected one review_resume call, got: ${describe(calls)}`,
+ };
}
export function assertFollowUpStaysConversational(output: unknown): AssertionResult {
diff --git a/evals/agent-chat/prompt.ts b/evals/agent-chat/prompt.ts
index 655c463..f65cc6b 100644
--- a/evals/agent-chat/prompt.ts
+++ b/evals/agent-chat/prompt.ts
@@ -8,17 +8,17 @@ type Message = Record<string, unknown>;
// across runs; the route generates a random one per request.
const NONCE = 'EVALNONCE0001';
-// Mirrors a completed get_resume round-trip so review rows can start after
-// the read, exactly as the model sees it on the second step of a turn.
-function resumeTurn(resumeText: string): Message[] {
+// Mirrors a completed review_resume round-trip so follow-up rows start after
+// the review, exactly as the model sees it on the second turn. The review
+// text is the tool's OUTPUT now, not the model's own prose.
+function reviewTurn(reviewBody: string): Message[] {
const output = {
status: 'ok',
resumeId: 'resume-eval-1',
title: 'Senior Engineer Resume',
- resumeText,
- chars: resumeText.length,
- truncated: false,
- source: 'default',
+ scores: { overall: 78, impact: 72, clarity: 81, atsCompatibility: 69 },
+ body: reviewBody,
+ saved: true,
};
return [
{
@@ -29,13 +29,13 @@ function resumeTurn(resumeText: string): Message[] {
reasoning_content: '',
tool_calls: [
{
- id: 'call_get_resume_1',
+ id: 'call_review_resume_1',
type: 'function',
- function: { name: 'get_resume', arguments: '{}' },
+ function: { name: 'review_resume', arguments: '{}' },
},
],
},
- { role: 'tool', tool_call_id: 'call_get_resume_1', content: JSON.stringify(output) },
+ { role: 'tool', tool_call_id: 'call_review_resume_1', content: JSON.stringify(output) },
];
}
@@ -51,8 +51,7 @@ export default function prompt({ vars }: { vars: Record<string, string> }): Mess
messages.push({ role: 'user', content: buildPasteContextMessage(block) });
}
- if (vars.resumeText) messages.push(...resumeTurn(vars.resumeText));
- if (vars.priorReview) messages.push({ role: 'assistant', content: vars.priorReview });
+ if (vars.priorReview) messages.push(...reviewTurn(vars.priorReview));
if (vars.followUp) messages.push({ role: 'user', content: vars.followUp });
return messages;
diff --git a/evals/agent-chat/promptfooconfig.yaml b/evals/agent-chat/promptfooconfig.yaml
index 97620b9..1236906 100644
--- a/evals/agent-chat/promptfooconfig.yaml
+++ b/evals/agent-chat/promptfooconfig.yaml
@@ -161,14 +161,13 @@ tests:
The reply states it cannot read the user's saved jobs. It does not
give a number and does not call a tool.
- # The review flow's first step. No resume text is supplied, so the only
- # correct move is to read one.
- - description: Review request - calls get_resume rather than inventing a review
+ # The review flow's only step now: one tool call, and the app does the rest.
+ - description: Review request - calls review_resume
vars:
userMessage: Review my resume.
assert:
- type: javascript
- value: file://./assertions.ts:assertCallsGetResume
+ value: file://./assertions.ts:assertCallsReviewResume
# get_resume is a general read, not a review tool. A plain question about
# resume contents should reach for it too.
@@ -179,23 +178,15 @@ tests:
- type: javascript
value: file://./assertions.ts:assertCallsGetResume
- # Second step of the turn: the read already happened, so what is measured
- # is the review itself, through the parser all three surfaces share.
- - description: After the read - produces a parseable SCORES line and a full review
+ # The two resume tools must not be interchangeable: a critique is a review,
+ # a factual question is a read. This is the row that catches a tool
+ # description that blurs them.
+ - description: Critique wording - still review_resume, not get_resume
vars:
- userMessage: Review my resume.
- resumeText: file://../resume-review/fixtures/strong-senior.txt
- assert:
- - type: javascript
- value: file://./assertions.ts:assertReviewParses
-
- - description: Sparse resume - still produces a parseable review, not a refusal
- vars:
- userMessage: Review my resume.
- resumeText: file://../resume-review/fixtures/weak-sparse.txt
+ userMessage: Can you give me feedback on how strong my resume is?
assert:
- type: javascript
- value: file://./assertions.ts:assertReviewParses
+ value: file://./assertions.ts:assertCallsReviewResume
# Follow-ups are the whole point of moving review into the chat: answered
# from context, no second read, and no second SCORES line to re-trigger a
@@ -203,7 +194,6 @@ tests:
- description: Follow-up question - answers from context, no re-read, no new scores
vars:
userMessage: Review my resume.
- resumeText: file://../resume-review/fixtures/strong-senior.txt
priorReview: file://./fixtures/prior-review.md
followUp: Which of those improvements would you do first, and why?
assert:
@@ -213,7 +203,6 @@ tests:
- description: Follow-up asking about a specific section - still conversational
vars:
userMessage: Review my resume.
- resumeText: file://../resume-review/fixtures/strong-senior.txt
priorReview: file://./fixtures/prior-review.md
followUp: Rewrite the summary bullet you flagged.
assert: