Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

8.8 KB
Raw Blame
import {
  AGENT_CHAT_PROMPT_SECTIONS,
  AGENT_CHAT_SYSTEM_PROMPT,
  AGENT_TOOL_DESCRIPTIONS,
  buildPasteContextMessage,
} from "@/lib/agent/prompt";
import { AgentAddJobSchema } from "@/models/agent.schema";
import {
  RESUME_REVIEW_SYSTEM_PROMPT,
  RESUME_REVIEW_OUTPUT_FORMAT,
} from "@/lib/ai/prompts/resume-review";

const SCORES_LINE = "SCORES: overall=<0-100> impact=<0-100> clarity=<0-100> ats=<0-100>";

describe("agent chat system prompt", () => {
  it("is composed from named sections, all of which appear in the prompt", () => {
    const keys = Object.keys(AGENT_CHAT_PROMPT_SECTIONS);
    expect(keys).toEqual(
      expect.arrayContaining(["role", "capabilities", "limitations", "paste", "safety"]),
    );
    for (const key of keys) {
      expect(AGENT_CHAT_SYSTEM_PROMPT).toContain(
        AGENT_CHAT_PROMPT_SECTIONS[key as keyof typeof AGENT_CHAT_PROMPT_SECTIONS],
      );
    }
  });

  it("keeps what the agent can do separate from what it cannot read", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toContain("add_job");
    expect(AGENT_CHAT_PROMPT_SECTIONS.limitations).toMatch(/cannot read/i);
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).not.toMatch(/cannot read/i);
  });

  // Measured, not assumed: without this, qwen3.5:9b extracted correctly and
  // then asked for confirmation in prose on 5 of 7 eval rows instead of
  // calling the tool, re-implementing the approval card in chat text.
  it("tells the model to call the tool instead of confirming in chat", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never ask.*confirm in the chat/i);
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never ask the user for optional fields/i);
  });

  // The approval-resume POST narrates a write that already happened, and the
  // only card language in the section is future-tense — so the model told the
  // user to "hit Confirm to save it" for a job already in the database.
  it("tells the model a created job is already saved", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/already pressed Confirm/i);
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never mention the confirmation card/i);
  });

  // The model has to CLASSIFY nothing here. Asking it to decide whether a
  // paste happened failed three ways in testing — it invented invisible
  // markers, then treated a missing marker as an anomaly to raise with the
  // user, and spent 18k characters of reasoning either way. execute prefers
  // pastedText over the model's field, so an unconditional rule is correct
  // in both cases and has no decision to get wrong.
  it("requires jobDescription unconditionally, with no paste branch to decide", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.paste).toMatch(/verbatim/i);
    expect(AGENT_CHAT_PROMPT_SECTIONS.paste).toMatch(/no case where you leave it out/i);
    // No instruction to omit, and nothing for the model to weigh up.
    expect(AGENT_CHAT_PROMPT_SECTIONS.paste).not.toMatch(/omit the field|omit this field|omit it entirely/i);
    expect(AGENT_CHAT_PROMPT_SECTIONS.paste).not.toContain("<<<PASTED_");
    // Say what to send, never what the app does with it. Told the app
    // substitutes values, qwen3.5 generalised that to every optional field
    // and filled all 16k of num_ctx deciding which ones it fills in.
    expect(AGENT_CHAT_PROMPT_SECTIONS.paste).not.toMatch(/the app (replaces|supplies|already has)/i);
  });

  // The description the model reads is the other half of the same rule; the
  // two disagreeing is what the model resolves by guessing.
  it("keeps the jobDescription schema description free of an omit instruction", () => {
    const described = AgentAddJobSchema.shape.jobDescription.description ?? "";
    expect(described).toMatch(/always include this field/i);
    expect(described).not.toMatch(/omit/i);
    expect(described).not.toMatch(/the app (replaces|supplies|already has)/i);
  });

  it("names the approval verbs the UI actually uses", () => {
    expect(AGENT_CHAT_SYSTEM_PROMPT).toContain("Confirm");
    expect(AGENT_CHAT_SYSTEM_PROMPT).toContain("Cancel");
  });

  it("treats delimited pasted content as data, never as instructions", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.safety).toMatch(/never.*instructions/i);
    expect(buildPasteContextMessage("<<<X>>>body<<<END X>>>")).toContain("<<<X>>>body<<<END X>>>");
    expect(buildPasteContextMessage("BLOCK")).toMatch(/data/i);
  });

  it("describes add_job without mentioning tools the chat does not expose", () => {
    expect(AGENT_TOOL_DESCRIPTIONS.add_job).toBeTruthy();
    expect(AGENT_TOOL_DESCRIPTIONS.add_job).not.toMatch(/upsert|allowDuplicate|update_job/);
  });

  // The review is generated by review_resume's own nested call now, so the
  // output format has no business in the chat prompt. Deleting it is the
  // point of the change, not a side effect: it was the longest block in the
  // prompt and the measured cause of qwen3.5:9b's add_job regression.
  it("no longer carries the review output format", () => {
    expect(AGENT_CHAT_SYSTEM_PROMPT).not.toContain(RESUME_REVIEW_OUTPUT_FORMAT);
    expect(AGENT_CHAT_SYSTEM_PROMPT).not.toContain("SCORES:");
    expect(RESUME_REVIEW_SYSTEM_PROMPT).toContain(RESUME_REVIEW_OUTPUT_FORMAT);
  });

  it("keeps the exact SCORES line the shared parser matches", () => {
    expect(RESUME_REVIEW_OUTPUT_FORMAT).toContain(SCORES_LINE);
    expect(RESUME_REVIEW_OUTPUT_FORMAT).toContain("## Grammar & Spelling");
  });

  it("no longer claims resumes are unreadable", () => {
    expect(AGENT_CHAT_SYSTEM_PROMPT).toContain("get_resume");
    expect(AGENT_CHAT_SYSTEM_PROMPT).not.toContain(
      "saved jobs, resumes, tasks",
    );
  });

  it("routes review requests to the tool instead of describing a review", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS).not.toHaveProperty("review");
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toContain("review_resume");
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never write a review yourself/i);
  });

  // Measured in eval-B74: told only that match_job "works on a job's page",
  // qwen3.5:9b asked the user to open one on 3 of 3 rows instead of calling it.
  // It cannot see the page, so the precondition is the app's to report.
  it("tells the model to call match_job rather than ask which page the user is on", () => {
    expect(AGENT_TOOL_DESCRIPTIONS.match_job).toMatch(/never ask them to open a job/i);
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/never ask them to open a job/i);
  });

  it("describes all five tools", () => {
    expect(Object.keys(AGENT_TOOL_DESCRIPTIONS)).toEqual([
      "add_job",
      "get_resume",
      "review_resume",
      "match_job",
      "generate_cover_letter",
    ]);
  });

  // Same lesson as match_job in eval-B74: a precondition the tool can check
  // for itself must never be stated as one the model has to verify first.
  it("tells the model to call generate_cover_letter rather than ask which page the user is on", () => {
    expect(AGENT_TOOL_DESCRIPTIONS.generate_cover_letter).toMatch(
      /never ask them to open a job/i,
    );
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(
      /never write the letter yourself/i,
    );
  });

  // The model replayed a turn-1 no_job result on turns 2 and 3, quoting the
  // prompt's own "you cannot see which page" claim as its reason. The claim
  // is false once every turn carries a page line, and it has to go.
  it("no longer claims the model cannot see which page the user is on", () => {
    expect(AGENT_CHAT_SYSTEM_PROMPT).not.toMatch(/cannot see which page/i);
    expect(AGENT_TOOL_DESCRIPTIONS.match_job).not.toMatch(/cannot tell which page/i);
    expect(AGENT_TOOL_DESCRIPTIONS.generate_cover_letter).not.toMatch(
      /cannot tell which page/i,
    );
  });

  it("tells the model an earlier no_job result is stale", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/no_job/);
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/earlier turn/i);
    expect(AGENT_TOOL_DESCRIPTIONS.match_job).toMatch(/stale/i);
    expect(AGENT_TOOL_DESCRIPTIONS.generate_cover_letter).toMatch(/stale/i);
  });

  // jobLookup returns no_job for an unresolvable id too, so the block saying
  // a job is open and this turn's tool saying no_job is a reachable state.
  it("still believes a no_job result from the current turn", () => {
    expect(AGENT_CHAT_PROMPT_SECTIONS.capabilities).toMatch(/on this turn is current/i);
  });

  // get_resume stays the on-ramp for reading a resume, but it is no longer
  // the review path — leaving the clause in gives the model two ways to
  // answer "review my resume" and it picks the wrong one.
  it("strips the review clause from the get_resume description", () => {
    expect(AGENT_TOOL_DESCRIPTIONS.get_resume).not.toMatch(/review/i);
    expect(AGENT_TOOL_DESCRIPTIONS.review_resume).toMatch(/review/i);
  });
});