Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
test(evals): run prompt regressions against local Ollama alongside DeepSeek
Khuram Niaz655a3cc007e1
diff --git a/evals/cover-letter/promptfooconfig.yaml b/evals/cover-letter/promptfooconfig.yamlindex e8be483..aa7cb56 100644--- a/evals/cover-letter/promptfooconfig.yaml+++ b/evals/cover-letter/promptfooconfig.yaml@@ -2,6 +2,7 @@ description: Cover letter prompt regression testsproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -9,6 +10,19 @@ providers:temperature: 0max_tokens: 8192+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in+ # passthrough to clear promptfoo's reasoning-model name check; disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 8192+ passthrough:+ reasoning_effort: "none"+prompts:- file://./prompt.tsdiff --git a/evals/dispatch-prompt.ts b/evals/dispatch-prompt.tsindex 81a4775..66948ae 100644--- a/evals/dispatch-prompt.ts+++ b/evals/dispatch-prompt.ts@@ -41,18 +41,22 @@ export default function prompt({ vars }: { vars: Record<string, string> }) {},];case 'resume-import':- // Import rows override the provider to json_object mode (see the combined- // config), mirroring the route's Output.object; append the JSON hint that- // DeepSeek's JSON mode requires.- return [- { role: 'system', content: RESUME_IMPORT_SYSTEM_PROMPT },- {- role: 'user',- content:- buildResumeImportPrompt(vars.resumeText) +- '\n\nReturn the result as a single JSON object.',- },- ];+ // Returning config alongside the prompt switches only these rows to+ // json_object mode, mirroring the route's Output.object, so every+ // provider stays markdown-mode for the other tasks. The trailing hint is+ // what DeepSeek's JSON mode requires.+ return {+ prompt: [+ { role: 'system', content: RESUME_IMPORT_SYSTEM_PROMPT },+ {+ role: 'user',+ content:+ buildResumeImportPrompt(vars.resumeText) ++ '\n\nReturn the result as a single JSON object.',+ },+ ],+ config: { response_format: { type: 'json_object' } },+ };default:throw new Error(`Unknown eval task: ${vars.task}`);}diff --git a/evals/job-match/automation.promptfooconfig.yaml b/evals/job-match/automation.promptfooconfig.yamlindex 8ad2ab0..a12633d 100644--- a/evals/job-match/automation.promptfooconfig.yaml+++ b/evals/job-match/automation.promptfooconfig.yaml@@ -2,6 +2,7 @@ description: Automation (lean) job match prompt regression testsproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -9,6 +10,19 @@ providers:temperature: 0max_tokens: 4096+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in+ # passthrough to clear promptfoo's reasoning-model name check; disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 4096+ passthrough:+ reasoning_effort: "none"+prompts:- file://./automation-prompt.tsdiff --git a/evals/job-match/promptfooconfig.yaml b/evals/job-match/promptfooconfig.yamlindex ed622cd..791d3c8 100644--- a/evals/job-match/promptfooconfig.yaml+++ b/evals/job-match/promptfooconfig.yaml@@ -2,6 +2,7 @@ description: Job match prompt regression testsproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -9,6 +10,19 @@ providers:temperature: 0max_tokens: 8192+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in+ # passthrough to clear promptfoo's reasoning-model name check; disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 8192+ passthrough:+ reasoning_effort: "none"+prompts:- file://./prompt.tsdiff --git a/evals/mcp-tools/promptfooconfig.yaml b/evals/mcp-tools/promptfooconfig.yamlindex 1610ebd..1641f04 100644--- a/evals/mcp-tools/promptfooconfig.yaml+++ b/evals/mcp-tools/promptfooconfig.yaml@@ -8,6 +8,7 @@ description: MCP tool-routing regression tests — which tool the agent picks, aproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -17,6 +18,21 @@ providers:tool_choice: autotools: file://./tools.json+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in+ # passthrough to clear promptfoo's reasoning-model name check; disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 8192+ passthrough:+ reasoning_effort: "none"+ tool_choice: auto+ tools: file://./tools.json+prompts:- file://./prompt.tsdiff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yamlindex 18b00a9..c9f5c48 100644--- a/evals/promptfooconfig.yaml+++ b/evals/promptfooconfig.yaml@@ -2,6 +2,7 @@ description: All prompt regression tests (resume-review + job-match + automationproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -9,6 +10,22 @@ providers:temperature: 0max_tokens: 8192+ # Local Ollama via its OpenAI-compatible endpoint, not promptfoo's native+ # ollama provider: only the openai provider merges the per-task config that+ # dispatch-prompt.ts returns for resume-import's JSON mode.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 8192+ # passthrough bypasses promptfoo's reasoning-model name check, which+ # would drop reasoning_effort for a non-OpenAI model name. Disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ passthrough:+ reasoning_effort: "none"+prompts:- file://./dispatch-prompt.ts@@ -218,20 +235,10 @@ tests:- "documents per second"- "documents/second"- # Resume import (structured JSON via Output.object). These rows override the- # provider to JSON mode; the shared markdown provider above can't be reused- # because it would force JSON onto the review/match tasks too.+ # Resume import (structured JSON via Output.object). JSON mode is set per-task+ # by dispatch-prompt.ts, not per-provider, so these rows run against every+ # provider above without forcing JSON onto the review/match tasks.- description: "[resume-import] well-structured resume"- provider:- id: openai:chat:deepseek-v4-flash- config:- apiBaseUrl: https://api.deepseek.com- apiKeyEnvar: DEEPSEEK_API_KEY- showThinking: false- temperature: 0- max_tokens: 8192- response_format:- type: json_objectvars:task: resume-importresumeText: file://./resume-import/fixtures/standard.txt@@ -252,16 +259,6 @@ tests:value: file://./resume-import/assertions.ts:assertStandardCurrentJob- description: "[resume-import] embedded prompt injection + unrecognized sections"- provider:- id: openai:chat:deepseek-v4-flash- config:- apiBaseUrl: https://api.deepseek.com- apiKeyEnvar: DEEPSEEK_API_KEY- showThinking: false- temperature: 0- max_tokens: 8192- response_format:- type: json_objectvars:task: resume-importresumeText: file://./resume-import/fixtures/adversarial-resume.txtdiff --git a/evals/resume-import/promptfooconfig.yaml b/evals/resume-import/promptfooconfig.yamlindex 1b5ae17..03c9a96 100644--- a/evals/resume-import/promptfooconfig.yaml+++ b/evals/resume-import/promptfooconfig.yaml@@ -2,6 +2,7 @@ description: Resume import prompt regression testsproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -11,6 +12,21 @@ providers:response_format:type: json_object+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in+ # passthrough to clear promptfoo's reasoning-model name check; disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 8192+ passthrough:+ reasoning_effort: "none"+ response_format:+ type: json_object+prompts:- file://./prompt.tsdiff --git a/evals/resume-review/promptfooconfig.yaml b/evals/resume-review/promptfooconfig.yamlindex 844e788..14ff2aa 100644--- a/evals/resume-review/promptfooconfig.yaml+++ b/evals/resume-review/promptfooconfig.yaml@@ -2,6 +2,7 @@ description: Resume review prompt regression testsproviders:- id: openai:chat:deepseek-v4-flash+ label: deepseek-v4-flashconfig:apiBaseUrl: https://api.deepseek.comapiKeyEnvar: DEEPSEEK_API_KEY@@ -9,6 +10,19 @@ providers:temperature: 0max_tokens: 8192+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in+ # passthrough to clear promptfoo's reasoning-model name check; disabling+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.+ - id: openai:chat:qwen3.5:9b+ label: ollama-qwen3.5-9b+ config:+ apiBaseUrl: http://localhost:11434/v1+ apiKey: ollama+ temperature: 0+ max_tokens: 8192+ passthrough:+ reasoning_effort: "none"+prompts:- file://./prompt.ts