Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
fix(ai): harden cover-letter/resume-import prompts and MCP schema wording
Khuram Niazf00fbee75f1f
diff --git a/__tests__/resumeImport.schema.spec.ts b/__tests__/resumeImport.schema.spec.tsindex 577bf3d..4e79c60 100644--- a/__tests__/resumeImport.schema.spec.ts+++ b/__tests__/resumeImport.schema.spec.ts@@ -258,4 +258,27 @@ describe("ResumeImportSchema", () => {expect(result.data.certifications).toHaveLength(1);}});++ it("normalizes { sectionName, content } objects in unrecognizedSections", () => {+ const result = ResumeImportSchema.safeParse({+ unrecognizedSections: [+ { sectionName: "PROJECTS", content: "Built a thing" },+ { sectionName: "AWARDS", content: "Won a thing" },+ ],+ });+ expect(result.success).toBe(true);+ if (result.success) {+ expect(result.data.unrecognizedSections).toEqual(["PROJECTS", "AWARDS"]);+ }+ });++ it("keeps recognizable unrecognizedSections when one element is an unknown shape", () => {+ const result = ResumeImportSchema.safeParse({+ unrecognizedSections: ["Projects", { content: "no name key" }, 42, "Awards"],+ });+ expect(result.success).toBe(true);+ if (result.success) {+ expect(result.data.unrecognizedSections).toEqual(["Projects", "Awards"]);+ }+ });});diff --git a/evals/cover-letter/assertions.ts b/evals/cover-letter/assertions.tsindex bcf61aa..7d85617 100644--- a/evals/cover-letter/assertions.ts+++ b/evals/cover-letter/assertions.ts@@ -118,10 +118,20 @@ export function assertNoInventedNumbers(m[0].replace(/,/g, ''),);+ // "95K" and "95,000" are the same figure written two ways, so a source+ // number carrying a K/M/B suffix also allows its expanded form. This is not+ // the same as allowing derived numbers — the prompt forbids computing+ // percentages or totals, and those stay caught.+ const SUFFIX_SCALE: Record<string, number> = { k: 1e3, m: 1e6, b: 1e9 };+ const scaledIn = (text: string) =>+ [...(text || '').matchAll(/(\d+(?:[.,]\d+)*)\s*([KkMmBb])\b/g)].map((m) =>+ String(Number(m[1].replace(/,/g, '')) * SUFFIX_SCALE[m[2].toLowerCase()]),+ );+const allowed = new Set([context.vars.resumeText, context.vars.jobDescription, context.vars.guidance].filter(Boolean)- .flatMap((source) => numbersIn(source)),+ .flatMap((source) => [...numbersIn(source), ...scaledIn(source)]),);const invented = [...new Set(numbersIn(clean(output)))].filter(diff --git a/evals/cover-letter/promptfooconfig.yaml b/evals/cover-letter/promptfooconfig.yamlindex aa7cb56..0fb71ba 100644--- a/evals/cover-letter/promptfooconfig.yaml+++ b/evals/cover-letter/promptfooconfig.yaml@@ -26,6 +26,24 @@ providers:prompts:- file://./prompt.ts+# Grading provider for every llm-rubric below. Declared once here rather than+# inline per assertion, so a new rubric can't silently inherit promptfoo's+# defaults for these two settings:+# max_tokens — the default 1024 is a shared budget, and DeepSeek's reasoning+# can consume all of it, leaving zero content and a "No output" grade.+# showThinking — defaults true, which prepends "Thinking: ...\n\n" to the+# grade and breaks llm-rubric's JSON extraction.+defaultTest:+ options:+ provider:+ id: openai:chat:deepseek-v4-flash+ config:+ apiBaseUrl: https://api.deepseek.com+ apiKeyEnvar: DEEPSEEK_API_KEY+ showThinking: false+ temperature: 0+ max_tokens: 4096+# The letter is generated at a creative temperature in the app (TEMPERATURES.# FEEDBACK), but evals run at 0 for repeatability — same trade-off as the other# markdown prompts here.@@ -56,19 +74,18 @@ tests:- "Go"- "AWS"- type: llm-rubric- provider:- id: openai:chat:deepseek-v4-flash- config:- apiBaseUrl: https://api.deepseek.com- apiKeyEnvar: DEEPSEEK_API_KEY- temperature: 0value: >-Every employer, job title, date, degree, certification, and metric inthe letter must appear in the candidate's resume below. Fail if theletter names any employer, title, school, degree, or certification the- resume does not contain, or states an achievement the resume does not- support. Company names taken from the job posting are acceptable when- used as the employer being applied to. Resume:+ resume does not contain, states an achievement the resume does not+ support, or credits one employer with an achievement the resume lists+ under a different employer. Company names taken from the job posting+ are acceptable when used as the employer being applied to. Judge only+ titles the letter claims the candidate has HELD at a named employer —+ a general self-description aligned to the role applied for ("I am a+ senior backend engineer") is not a title claim and is acceptable.+ Resume:{{resumeText}}diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yamlindex c9f5c48..e076231 100644--- a/evals/promptfooconfig.yaml+++ b/evals/promptfooconfig.yaml@@ -29,6 +29,24 @@ providers:prompts:- file://./dispatch-prompt.ts+# Grading provider for every llm-rubric below. Declared once here rather than+# inline per assertion, so a new rubric can't silently inherit promptfoo's+# defaults for these two settings:+# max_tokens — the default 1024 is a shared budget, and DeepSeek's reasoning+# can consume all of it, leaving zero content and a "No output" grade.+# showThinking — defaults true, which prepends "Thinking: ...\n\n" to the+# grade and breaks llm-rubric's JSON extraction.+defaultTest:+ options:+ provider:+ id: openai:chat:deepseek-v4-flash+ config:+ apiBaseUrl: https://api.deepseek.com+ apiKeyEnvar: DEEPSEEK_API_KEY+ showThinking: false+ temperature: 0+ max_tokens: 4096+tests:# Resume review- description: "[resume-review] strong senior resume"@@ -189,19 +207,18 @@ tests:- "Go"- "AWS"- type: llm-rubric- provider:- id: openai:chat:deepseek-v4-flash- config:- apiBaseUrl: https://api.deepseek.com- apiKeyEnvar: DEEPSEEK_API_KEY- temperature: 0value: >-Every employer, job title, date, degree, certification, and metric inthe letter must appear in the candidate's resume below. Fail if theletter names any employer, title, school, degree, or certification the- resume does not contain, or states an achievement the resume does not- support. Company names taken from the job posting are acceptable when- used as the employer being applied to. Resume:+ resume does not contain, states an achievement the resume does not+ support, or credits one employer with an achievement the resume lists+ under a different employer. Company names taken from the job posting+ are acceptable when used as the employer being applied to. Judge only+ titles the letter claims the candidate has HELD at a named employer —+ a general self-description aligned to the role applied for ("I am a+ senior backend engineer") is not a title claim and is acceptable.+ Resume:{{resumeText}}diff --git a/src/lib/ai/prompts/cover-letter/system.ts b/src/lib/ai/prompts/cover-letter/system.tsindex 6a5c1d5..eac8dd6 100644--- a/src/lib/ai/prompts/cover-letter/system.ts+++ b/src/lib/ai/prompts/cover-letter/system.ts@@ -9,6 +9,10 @@ export const COVER_LETTER_SYSTEM_PROMPT = `You are an experienced career writer## RULES- Write in the candidate's voice, first person, present tense where natural.- Ground every claim in the resume. Never invent employers, titles, dates, degrees, certifications, or metrics that are not in the resume.+- Cite figures exactly as the resume states them. Any figure describing the candidate's own work must come from the resume; a figure taken from the job posting stays attributed to the company and is never restated as the candidate's result. Do NOT compute, derive, or convert numbers — no percentages from before/after values, no totals, no ratios, no rounding. "reduced p99 latency from 820ms to 140ms" must not become "an 83% reduction". If a figure is not literally in the inputs, describe the result in words instead.+- Before citing any achievement, find which employer the resume lists it under and keep it there. If a sentence names an employer, every achievement in that sentence must come from that employer's bullets — do not pull in a bullet from a different role, and do not merge two roles into one claim. When in doubt, cite the achievement without naming an employer.+- Never restate something at a larger scale or scope than the resume gives it. An 18TB database is not "petabyte scale"; leading one migration is not "leading all platform work".+- A PRIOR MATCH ANALYSIS block, when present, sets emphasis only — it is not a source of facts. Use a keyword or tip only where the resume already supports it, and drop the rest rather than stretching to cover the list.- Address the job's actual stated requirements, not generic enthusiasm.- 250-400 words total. Three to five short paragraphs.- Confident and specific. No filler ("I am writing to apply for..."), no flattery of the company, no restating the whole resume.@@ -29,6 +33,6 @@ Dear Hiring Manager,<closing: brief forward-looking sentence and a thank you>Sincerely,-<candidate name from the resume, or "Kind regards," with no name if the resume has none>+<candidate's full name exactly as the resume gives it; omit this line entirely if the resume has no name>Do NOT output JSON. Do NOT wrap the response in code fences. Do NOT add commentary, notes, or a preamble before or after the letter. Do NOT include placeholder brackets like [Company] — use real values from the inputs or omit the sentence.`;diff --git a/src/lib/ai/prompts/resume-import/system.ts b/src/lib/ai/prompts/resume-import/system.tsindex 7ce6ec1..f4496ea 100644--- a/src/lib/ai/prompts/resume-import/system.ts+++ b/src/lib/ai/prompts/resume-import/system.ts@@ -16,7 +16,7 @@ Never substitute a synonym (e.g. "title" for "jobTitle", "url" for- education: academic history with institution, degree, fieldOfStudy, location, startDate, endDate,description- certifications: licenses and certifications with title, organization, issueDate, expirationDate, credentialUrl-- unrecognizedSections: section names whose content cannot be mapped to any of the above fields (e.g. Projects, Publications, Volunteer Work, Awards). Do NOT include sections whose content was successfully parsed into contactInfo, summary, experience, education, certifications, or skills — even if the heading combines multiple categories (e.g. "Education & Certifications" or "Experience & Projects").+- unrecognizedSections: a flat array of plain section-name STRINGS whose content cannot be mapped to any of the above fields (e.g. ["Projects", "Publications", "Volunteer Work", "Awards"]). Names only — never objects, and never the section's content. Do NOT include sections whose content was successfully parsed into contactInfo, summary, experience, education, certifications, or skills — even if the heading combines multiple categories (e.g. "Education & Certifications" or "Experience & Projects").FIELD PRIORITY — for every experience and education entry, in this order:1. company/institution and jobTitle/degreediff --git a/src/lib/mcp/toolDescriptions.ts b/src/lib/mcp/toolDescriptions.tsindex 8a32751..5c27abe 100644--- a/src/lib/mcp/toolDescriptions.ts+++ b/src/lib/mcp/toolDescriptions.ts@@ -6,7 +6,7 @@ export const MCP_TOOL_DESCRIPTIONS = {add_job:"Add a job application to JobSync. Resolves or creates company, job title, location, and source by name. Returns a transparency report of what was matched vs. created.",find_job:- "Look up whether a job posting URL is already saved in JobSync. Call this before add_job when re-running a search, so an already-tracked posting is updated instead of duplicated.",+ "Look up whether a job posting is already saved in JobSync, by URL. Call this before add_job when re-running a search. With no URL to look up, use add_job's upsert instead of skipping dedupe.",update_job:"Correct or enrich a job previously added through MCP. Only the fields you supply change. Supplying a fuller jobDescription re-classifies the posting and requests a fresh match analysis — use this instead of re-adding with allowDuplicate.",add_question:diff --git a/src/models/mcp.schema.ts b/src/models/mcp.schema.tsindex 7fa9b61..5d84d5a 100644--- a/src/models/mcp.schema.ts+++ b/src/models/mcp.schema.ts@@ -37,18 +37,13 @@ export const McpAddJobInputShape = {.boolean().optional().describe(- "Force-create even if a matching job already exists. Prefer upsert:true " +- "for re-runs of the same search; use this only for a genuinely " +- "different posting.",+ "Force-create even if a matching job already exists. Prefer upsert:true for re-runs of the same search; use this only for a genuinely different posting.",),upsert: z.boolean().optional().describe(- "If a job with this URL (or same company+title within the dedupe " +- "window) already exists, update it with the fields supplied here " +- "instead of reporting a duplicate. Recommended for scheduled searches " +- "that resurface the same posting daily.",+ "If this posting is already saved, update it with the fields supplied here instead of reporting a duplicate. A posting counts as already saved when it matches on URL, or on company+title within the dedupe window — so this works for postings with no URL at all. Set this on every re-run of a saved or scheduled search, whether or not you have a URL.",),};@@ -85,12 +80,10 @@ export const McpSaveMatchResultInputShape = {.min(1).optional().describe(- "The id of the resume this match was scored against, exactly as given " +- "in the add_job directive. Omit only if the directive had none.",+ "The id of the resume this match was scored against, exactly as given in the add_job directive. Omit only if the directive had none.",),matchText: z.string().min(20).describe(- "Your full match analysis: a leading 'SCORES: match=<0-100> " +- "recommendation=<strong|good|partial|weak>' line, then a markdown body.",+ "Your full match analysis: a leading 'SCORES: match=<0-100> recommendation=<strong|good|partial|weak>' line, then a markdown body.",),};@@ -109,12 +102,10 @@ export const McpSaveResumeReviewInputShape = {.string().min(1).describe(- "The id of the resume this review was produced for, exactly as given " +- "in the review_resume directive.",+ "The id of the resume this review was produced for, exactly as given in the review_resume directive.",),reviewText: z.string().min(20).describe(- "Your full resume review: a leading 'SCORES: overall=<0-100> " +- "impact=<0-100> clarity=<0-100> ats=<0-100>' line, then a markdown body.",+ "Your full resume review: a leading 'SCORES: overall=<0-100> impact=<0-100> clarity=<0-100> ats=<0-100>' line, then a markdown body.",),};@@ -132,9 +123,7 @@ export const McpFindJobInputShape = {.string().url().describe(- "Direct URL to the job posting. Matched against saved jobs using the " +- "same canonical key as add_job's duplicate detection, so tracking " +- "parameters and host casing don't matter.",+ "Direct URL to the job posting. Matched against saved jobs using the same canonical key as add_job's duplicate detection, so tracking parameters and host casing don't matter.",),};@@ -148,8 +137,7 @@ export const McpUpdateJobInputShape = {.string().min(1).describe(- "The id of the job to update, as returned by add_job or find_job. " +- "Only jobs created through MCP can be updated.",+ "The id of the job to update, as returned by add_job or find_job. Only jobs created through MCP can be updated.",),company: z.string().min(1).optional(),jobTitle: z.string().min(1).optional(),@@ -158,9 +146,7 @@ export const McpUpdateJobInputShape = {.min(10).optional().describe(- "The complete job posting text, copied in full — do not summarize. " +- "Supplying this re-classifies the job's description completeness and, " +- "if it is now substantive enough, a fresh match analysis is requested.",+ "The complete job posting text, copied in full — do not summarize. Supplying this re-classifies the job's description completeness and, if it is now substantive enough, a fresh match analysis is requested.",),location: z.string().optional(),source: z.string().optional(),@@ -185,9 +171,7 @@ export const McpUpdateJobInputShape = {.array(z.string()).optional().describe(- "Skills required for the job, e.g. ['React', 'TypeScript']. Replaces " +- "the job's existing tags wholesale rather than merging, so include " +- "the tags returned by find_job that should be kept (max 10 applied).",+ "Skills required for the job, e.g. ['React', 'TypeScript']. Replaces the job's existing tags wholesale rather than merging, so include the tags returned by find_job that should be kept (max 10 applied).",),};@@ -207,9 +191,7 @@ export const McpAddJobsBatchInputShape = {.min(1).max(APP_CONSTANTS.MCP_BATCH_MAX_ITEMS).describe(- `Up to ${APP_CONSTANTS.MCP_BATCH_MAX_ITEMS} jobs, each with the same ` +- `fields as add_job. Processed in order; each item consumes one unit ` +- `of the MCP rate-limit budget.`,+ `Up to ${APP_CONSTANTS.MCP_BATCH_MAX_ITEMS} jobs, each with the same fields as add_job. Processed in order; each item consumes one unit of the MCP rate-limit budget.`,),};@@ -224,8 +206,7 @@ export const McpSaveMatchResultsBatchInputShape = {.min(1).max(APP_CONSTANTS.MCP_BATCH_MAX_ITEMS).describe(- `Up to ${APP_CONSTANTS.MCP_BATCH_MAX_ITEMS} match analyses, each with ` +- `the same fields as save_match_result.`,+ `Up to ${APP_CONSTANTS.MCP_BATCH_MAX_ITEMS} match analyses, each with the same fields as save_match_result.`,),};diff --git a/src/models/resumeImport.schema.ts b/src/models/resumeImport.schema.tsindex 0a86031..f8df96e 100644--- a/src/models/resumeImport.schema.ts+++ b/src/models/resumeImport.schema.ts@@ -58,6 +58,22 @@ export const ImportSkillsSchema = z.object({confidence,});+// One unrecognized section, as a plain name. Unknown shapes collapse to ""+// rather than throwing, so a single odd element can't void the whole array.+const unrecognizedSectionName = z+ .union([+ z.string(),+ z+ .object({+ name: z.string().optional(),+ sectionName: z.string().optional(),+ title: z.string().optional(),+ heading: z.string().optional(),+ })+ .transform((o) => o.name ?? o.sectionName ?? o.title ?? o.heading ?? ""),+ ])+ .catch("");+export const ResumeImportSchema = z.object({contactInfo: ImportContactInfoSchema.optional(),summary: z.string().catch(""),@@ -67,16 +83,13 @@ export const ResumeImportSchema = z.object({experience: z.array(ImportExperienceSchema).default([]),education: z.array(ImportEducationSchema).default([]),certifications: z.array(ImportCertificationSchema).default([]),- // Models inconsistently return strings or { name } objects here — accept both- // and normalize to strings. .catch keeps this non-critical field from ever- // failing whole-object validation.+ // Models inconsistently return strings or objects here, keying the name as+ // name/sectionName/title/heading. Normalize per element and drop only what+ // yields no name — an array-level .catch turned one unknown shape into a+ // silent [], so the user was never told content had been dropped.unrecognizedSections: z- .array(- z.union([- z.string(),- z.object({ name: z.string() }).transform((o) => o.name),- ]),- )+ .array(unrecognizedSectionName)+ .transform((names) => names.filter(Boolean)).catch([]).default([]),});