Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
test: retarget the agent-chat evals at review_resume selection
Khuram Niazafb2ceed5cbf
diff --git a/evals/agent-chat/assertions.ts b/evals/agent-chat/assertions.tsindex f1797ac..7dd5d2d 100644--- a/evals/agent-chat/assertions.ts+++ b/evals/agent-chat/assertions.ts@@ -123,17 +123,16 @@ export function assertCallsGetResume(output: unknown): AssertionResult {return { pass, score: pass ? 1 : 0, reason: pass ? 'called get_resume' : `expected one get_resume call, got: ${describe(calls)}` };}-// The three parse paths share this function, so passing here means the chat-// transcript, the review sheet and the MCP handler all read it identically.-export function assertReviewParses(output: unknown): AssertionResult {- const { scores, body } = parseResumeReview(textOf(output));- if (!scores) {- return { pass: false, score: 0, reason: 'no parseable SCORES line in the review' };- }- const inRange = [scores.overall, scores.impact, scores.clarity, scores.atsCompatibility].every((n) => n >= 0 && n <= 100);- const long = body.length >= 400;- const pass = inRange && long;- return { pass, score: pass ? 1 : 0, reason: pass ? `parsed overall=${scores.overall}, body ${body.length} chars` : `scores parsed but ${!inRange ? 'out of range' : `body only ${body.length} chars`}` };+export function assertCallsReviewResume(output: unknown): AssertionResult {+ const calls = parseToolCalls(output);+ const pass = calls.length === 1 && calls[0].name === 'review_resume';+ return {+ pass,+ score: pass ? 1 : 0,+ reason: pass+ ? 'called review_resume'+ : `expected one review_resume call, got: ${describe(calls)}`,+ };}export function assertFollowUpStaysConversational(output: unknown): AssertionResult {diff --git a/evals/agent-chat/prompt.ts b/evals/agent-chat/prompt.tsindex 655c463..f65cc6b 100644--- a/evals/agent-chat/prompt.ts+++ b/evals/agent-chat/prompt.ts@@ -8,17 +8,17 @@ type Message = Record<string, unknown>;// across runs; the route generates a random one per request.const NONCE = 'EVALNONCE0001';-// Mirrors a completed get_resume round-trip so review rows can start after-// the read, exactly as the model sees it on the second step of a turn.-function resumeTurn(resumeText: string): Message[] {+// Mirrors a completed review_resume round-trip so follow-up rows start after+// the review, exactly as the model sees it on the second turn. The review+// text is the tool's OUTPUT now, not the model's own prose.+function reviewTurn(reviewBody: string): Message[] {const output = {status: 'ok',resumeId: 'resume-eval-1',title: 'Senior Engineer Resume',- resumeText,- chars: resumeText.length,- truncated: false,- source: 'default',+ scores: { overall: 78, impact: 72, clarity: 81, atsCompatibility: 69 },+ body: reviewBody,+ saved: true,};return [{@@ -29,13 +29,13 @@ function resumeTurn(resumeText: string): Message[] {reasoning_content: '',tool_calls: [{- id: 'call_get_resume_1',+ id: 'call_review_resume_1',type: 'function',- function: { name: 'get_resume', arguments: '{}' },+ function: { name: 'review_resume', arguments: '{}' },},],},- { role: 'tool', tool_call_id: 'call_get_resume_1', content: JSON.stringify(output) },+ { role: 'tool', tool_call_id: 'call_review_resume_1', content: JSON.stringify(output) },];}@@ -51,8 +51,7 @@ export default function prompt({ vars }: { vars: Record<string, string> }): Messmessages.push({ role: 'user', content: buildPasteContextMessage(block) });}- if (vars.resumeText) messages.push(...resumeTurn(vars.resumeText));- if (vars.priorReview) messages.push({ role: 'assistant', content: vars.priorReview });+ if (vars.priorReview) messages.push(...reviewTurn(vars.priorReview));if (vars.followUp) messages.push({ role: 'user', content: vars.followUp });return messages;diff --git a/evals/agent-chat/promptfooconfig.yaml b/evals/agent-chat/promptfooconfig.yamlindex 97620b9..1236906 100644--- a/evals/agent-chat/promptfooconfig.yaml+++ b/evals/agent-chat/promptfooconfig.yaml@@ -161,14 +161,13 @@ tests:The reply states it cannot read the user's saved jobs. It does notgive a number and does not call a tool.- # The review flow's first step. No resume text is supplied, so the only- # correct move is to read one.- - description: Review request - calls get_resume rather than inventing a review+ # The review flow's only step now: one tool call, and the app does the rest.+ - description: Review request - calls review_resumevars:userMessage: Review my resume.assert:- type: javascript- value: file://./assertions.ts:assertCallsGetResume+ value: file://./assertions.ts:assertCallsReviewResume# get_resume is a general read, not a review tool. A plain question about# resume contents should reach for it too.@@ -179,23 +178,15 @@ tests:- type: javascriptvalue: file://./assertions.ts:assertCallsGetResume- # Second step of the turn: the read already happened, so what is measured- # is the review itself, through the parser all three surfaces share.- - description: After the read - produces a parseable SCORES line and a full review+ # The two resume tools must not be interchangeable: a critique is a review,+ # a factual question is a read. This is the row that catches a tool+ # description that blurs them.+ - description: Critique wording - still review_resume, not get_resumevars:- userMessage: Review my resume.- resumeText: file://../resume-review/fixtures/strong-senior.txt- assert:- - type: javascript- value: file://./assertions.ts:assertReviewParses-- - description: Sparse resume - still produces a parseable review, not a refusal- vars:- userMessage: Review my resume.- resumeText: file://../resume-review/fixtures/weak-sparse.txt+ userMessage: Can you give me feedback on how strong my resume is?assert:- type: javascript- value: file://./assertions.ts:assertReviewParses+ value: file://./assertions.ts:assertCallsReviewResume# Follow-ups are the whole point of moving review into the chat: answered# from context, no second read, and no second SCORES line to re-trigger a@@ -203,7 +194,6 @@ tests:- description: Follow-up question - answers from context, no re-read, no new scoresvars:userMessage: Review my resume.- resumeText: file://../resume-review/fixtures/strong-senior.txtpriorReview: file://./fixtures/prior-review.mdfollowUp: Which of those improvements would you do first, and why?assert:@@ -213,7 +203,6 @@ tests:- description: Follow-up asking about a specific section - still conversationalvars:userMessage: Review my resume.- resumeText: file://../resume-review/fixtures/strong-senior.txtpriorReview: file://./fixtures/prior-review.mdfollowUp: Rewrite the summary bullet you flagged.assert: