Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
import {
AGENT_CHAT_PROMPT_SECTIONS,
AGENT_CHAT_SYSTEM_PROMPT,
AGENT_TOOL_DESCRIPTIONS,
buildPasteContextMessage,
} from "@/lib/agent/prompt";
import { AgentAddJobSchema } from "@/models/agent.schema";
import {
RESUME_REVIEW_SYSTEM_PROMPT,
RESUME_REVIEW_OUTPUT_FORMAT,
} from "@/lib/ai/prompts/resume-review";
const SCORES_LINE = "SCORES: overall=<0-100> impact=<0-100> clarity=<0-100> ats=<0-100>";
describe("agent chat system prompt", () => {
it("is composed from named sections, all of which appear in the prompt", () => {
const keys = Object.keys(AGENT_CHAT_PROMPT_SECTIONS);
expect(keys).toEqual(
expect.arrayContaining(["role", "capabilities", "limitations", "paste", "safety"]),
);
for (const key of keys) {
expect(AGENT_CHAT_SYSTEM_PROMPT).toContain(
AGENT_CHAT_PROMPT_SECTIONS[key as keyof typeof AGENT_CHAT_PROMPT_SECTIONS],
);
}
});
it("keeps what the agent can do separate from what it cannot read", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toContain("add_job");
expect(AGENT_CHAT_PROMPT_SECTIONS.limitations).toMatch(/cannot read/i);
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).not.toMatch(/cannot read/i);
});
// Measured, not assumed: without this, qwen3.5:9b extracted correctly and
// then asked for confirmation in prose on 5 of 7 eval rows instead of
// calling the tool, re-implementing the approval card in chat text.
it("tells the model to call the tool instead of confirming in chat", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never ask.*confirm in the chat/i);
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never ask the user for optional fields/i);
});
// The approval-resume POST narrates a write that already happened, and the
// only card language in the section is future-tense — so the model told the
// user to "hit Confirm to save it" for a job already in the database.
it("tells the model a created job is already saved", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/already pressed Confirm/i);
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never mention the confirmation card/i);
});
// The model has to CLASSIFY nothing here. Asking it to decide whether a
// paste happened failed three ways in testing — it invented invisible
// markers, then treated a missing marker as an anomaly to raise with the
// user, and spent 18k characters of reasoning either way. execute prefers
// pastedText over the model's field, so an unconditional rule is correct
// in both cases and has no decision to get wrong.
it("requires jobDescription unconditionally, with no paste branch to decide", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.paste).toMatch(/verbatim/i);
expect(AGENT_CHAT_PROMPT_SECTIONS.paste).toMatch(/no case where you leave it out/i);
// No instruction to omit, and nothing for the model to weigh up.
expect(AGENT_CHAT_PROMPT_SECTIONS.paste).not.toMatch(/omit the field|omit this field|omit it entirely/i);
expect(AGENT_CHAT_PROMPT_SECTIONS.paste).not.toContain("<<<PASTED_");
// Say what to send, never what the app does with it. Told the app
// substitutes values, qwen3.5 generalised that to every optional field
// and filled all 16k of num_ctx deciding which ones it fills in.
expect(AGENT_CHAT_PROMPT_SECTIONS.paste).not.toMatch(/the app (replaces|supplies|already has)/i);
});
// The description the model reads is the other half of the same rule; the
// two disagreeing is what the model resolves by guessing.
it("keeps the jobDescription schema description free of an omit instruction", () => {
const described = AgentAddJobSchema.shape.jobDescription.description ?? "";
expect(described).toMatch(/always include this field/i);
expect(described).not.toMatch(/omit/i);
expect(described).not.toMatch(/the app (replaces|supplies|already has)/i);
});
it("names the approval verbs the UI actually uses", () => {
expect(AGENT_CHAT_SYSTEM_PROMPT).toContain("Confirm");
expect(AGENT_CHAT_SYSTEM_PROMPT).toContain("Cancel");
});
it("treats delimited pasted content as data, never as instructions", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.safety).toMatch(/never.*instructions/i);
expect(buildPasteContextMessage("<<<X>>>body<<<END X>>>")).toContain("<<<X>>>body<<<END X>>>");
expect(buildPasteContextMessage("BLOCK")).toMatch(/data/i);
});
it("describes add_job without mentioning tools the chat does not expose", () => {
expect(AGENT_TOOL_DESCRIPTIONS.add_job).toBeTruthy();
expect(AGENT_TOOL_DESCRIPTIONS.add_job).not.toMatch(/upsert|allowDuplicate|update_job/);
});
// The review is generated by review_resume's own nested call now, so the
// output format has no business in the chat prompt. Deleting it is the
// point of the change, not a side effect: it was the longest block in the
// prompt and the measured cause of qwen3.5:9b's add_job regression.
it("no longer carries the review output format", () => {
expect(AGENT_CHAT_SYSTEM_PROMPT).not.toContain(RESUME_REVIEW_OUTPUT_FORMAT);
expect(AGENT_CHAT_SYSTEM_PROMPT).not.toContain("SCORES:");
expect(RESUME_REVIEW_SYSTEM_PROMPT).toContain(RESUME_REVIEW_OUTPUT_FORMAT);
});
it("keeps the exact SCORES line the shared parser matches", () => {
expect(RESUME_REVIEW_OUTPUT_FORMAT).toContain(SCORES_LINE);
expect(RESUME_REVIEW_OUTPUT_FORMAT).toContain("## Grammar & Spelling");
});
it("no longer claims resumes are unreadable", () => {
expect(AGENT_CHAT_SYSTEM_PROMPT).toContain("get_resume");
expect(AGENT_CHAT_SYSTEM_PROMPT).not.toContain(
"saved jobs, resumes, tasks",
);
});
it("routes review requests to the tool instead of describing a review", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS).not.toHaveProperty("review");
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toContain("review_resume");
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never write a review yourself/i);
});
// Measured in eval-B74: told only that match_job "works on a job's page",
// qwen3.5:9b asked the user to open one on 3 of 3 rows instead of calling it.
// It cannot see the page, so the precondition is the app's to report.
it("tells the model to call match_job rather than ask which page the user is on", () => {
expect(AGENT_TOOL_DESCRIPTIONS.match_job).toMatch(/never ask them to open a job/i);
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never ask them to open a job/i);
});
it("describes all five tools", () => {
expect(Object.keys(AGENT_TOOL_DESCRIPTIONS)).toEqual([
"add_job",
"get_resume",
"review_resume",
"match_job",
"generate_cover_letter",
]);
});
// Same lesson as match_job in eval-B74: a precondition the tool can check
// for itself must never be stated as one the model has to verify first.
it("tells the model to call generate_cover_letter rather than ask which page the user is on", () => {
expect(AGENT_TOOL_DESCRIPTIONS.generate_cover_letter).toMatch(
/never ask them to open a job/i,
);
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(
/never write the letter yourself/i,
);
});
// The model replayed a turn-1 no_job result on turns 2 and 3, quoting the
// prompt's own "you cannot see which page" claim as its reason. The claim
// is false once every turn carries a page line, and it has to go.
it("no longer claims the model cannot see which page the user is on", () => {
expect(AGENT_CHAT_SYSTEM_PROMPT).not.toMatch(/cannot see which page/i);
expect(AGENT_TOOL_DESCRIPTIONS.match_job).not.toMatch(/cannot tell which page/i);
expect(AGENT_TOOL_DESCRIPTIONS.generate_cover_letter).not.toMatch(
/cannot tell which page/i,
);
});
it("tells the model an earlier no_job result is stale", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/no_job/);
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/earlier turn/i);
expect(AGENT_TOOL_DESCRIPTIONS.match_job).toMatch(/stale/i);
expect(AGENT_TOOL_DESCRIPTIONS.generate_cover_letter).toMatch(/stale/i);
});
// jobLookup returns no_job for an unresolvable id too, so the block saying
// a job is open and this turn's tool saying no_job is a reachable state.
it("still believes a no_job result from the current turn", () => {
expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/on this turn is current/i);
});
// get_resume stays the on-ramp for reading a resume, but it is no longer
// the review path — leaving the clause in gives the model two ways to
// answer "review my resume" and it picks the wrong one.
it("strips the review clause from the get_resume description", () => {
expect(AGENT_TOOL_DESCRIPTIONS.get_resume).not.toMatch(/review/i);
expect(AGENT_TOOL_DESCRIPTIONS.review_resume).toMatch(/review/i);
});
});