Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

test(evals): run prompt regressions against local Ollama alongside DeepSeek

Khuram Niaz655a3cc007e1
diff --git a/evals/cover-letter/promptfooconfig.yaml b/evals/cover-letter/promptfooconfig.yaml
index e8be483..aa7cb56 100644
--- a/evals/cover-letter/promptfooconfig.yaml
+++ b/evals/cover-letter/promptfooconfig.yaml
@@ -2,6 +2,7 @@ description: Cover letter prompt regression tests
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -9,6 +10,19 @@ providers:
temperature: 0
max_tokens: 8192
+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in
+ # passthrough to clear promptfoo's reasoning-model name check; disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 8192
+ passthrough:
+ reasoning_effort: "none"
+
prompts:
- file://./prompt.ts
diff --git a/evals/dispatch-prompt.ts b/evals/dispatch-prompt.ts
index 81a4775..66948ae 100644
--- a/evals/dispatch-prompt.ts
+++ b/evals/dispatch-prompt.ts
@@ -41,18 +41,22 @@ export default function prompt({ vars }: { vars: Record<string, string> }) {
},
];
case 'resume-import':
- // Import rows override the provider to json_object mode (see the combined
- // config), mirroring the route's Output.object; append the JSON hint that
- // DeepSeek's JSON mode requires.
- return [
- { role: 'system', content: RESUME_IMPORT_SYSTEM_PROMPT },
- {
- role: 'user',
- content:
- buildResumeImportPrompt(vars.resumeText) +
- '\n\nReturn the result as a single JSON object.',
- },
- ];
+ // Returning config alongside the prompt switches only these rows to
+ // json_object mode, mirroring the route's Output.object, so every
+ // provider stays markdown-mode for the other tasks. The trailing hint is
+ // what DeepSeek's JSON mode requires.
+ return {
+ prompt: [
+ { role: 'system', content: RESUME_IMPORT_SYSTEM_PROMPT },
+ {
+ role: 'user',
+ content:
+ buildResumeImportPrompt(vars.resumeText) +
+ '\n\nReturn the result as a single JSON object.',
+ },
+ ],
+ config: { response_format: { type: 'json_object' } },
+ };
default:
throw new Error(`Unknown eval task: ${vars.task}`);
}
diff --git a/evals/job-match/automation.promptfooconfig.yaml b/evals/job-match/automation.promptfooconfig.yaml
index 8ad2ab0..a12633d 100644
--- a/evals/job-match/automation.promptfooconfig.yaml
+++ b/evals/job-match/automation.promptfooconfig.yaml
@@ -2,6 +2,7 @@ description: Automation (lean) job match prompt regression tests
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -9,6 +10,19 @@ providers:
temperature: 0
max_tokens: 4096
+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in
+ # passthrough to clear promptfoo's reasoning-model name check; disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 4096
+ passthrough:
+ reasoning_effort: "none"
+
prompts:
- file://./automation-prompt.ts
diff --git a/evals/job-match/promptfooconfig.yaml b/evals/job-match/promptfooconfig.yaml
index ed622cd..791d3c8 100644
--- a/evals/job-match/promptfooconfig.yaml
+++ b/evals/job-match/promptfooconfig.yaml
@@ -2,6 +2,7 @@ description: Job match prompt regression tests
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -9,6 +10,19 @@ providers:
temperature: 0
max_tokens: 8192
+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in
+ # passthrough to clear promptfoo's reasoning-model name check; disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 8192
+ passthrough:
+ reasoning_effort: "none"
+
prompts:
- file://./prompt.ts
diff --git a/evals/mcp-tools/promptfooconfig.yaml b/evals/mcp-tools/promptfooconfig.yaml
index 1610ebd..1641f04 100644
--- a/evals/mcp-tools/promptfooconfig.yaml
+++ b/evals/mcp-tools/promptfooconfig.yaml
@@ -8,6 +8,7 @@ description: MCP tool-routing regression tests — which tool the agent picks, a
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -17,6 +18,21 @@ providers:
tool_choice: auto
tools: file://./tools.json
+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in
+ # passthrough to clear promptfoo's reasoning-model name check; disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 8192
+ passthrough:
+ reasoning_effort: "none"
+ tool_choice: auto
+ tools: file://./tools.json
+
prompts:
- file://./prompt.ts
diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml
index 18b00a9..c9f5c48 100644
--- a/evals/promptfooconfig.yaml
+++ b/evals/promptfooconfig.yaml
@@ -2,6 +2,7 @@ description: All prompt regression tests (resume-review + job-match + automation
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -9,6 +10,22 @@ providers:
temperature: 0
max_tokens: 8192
+ # Local Ollama via its OpenAI-compatible endpoint, not promptfoo's native
+ # ollama provider: only the openai provider merges the per-task config that
+ # dispatch-prompt.ts returns for resume-import's JSON mode.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 8192
+ # passthrough bypasses promptfoo's reasoning-model name check, which
+ # would drop reasoning_effort for a non-OpenAI model name. Disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ passthrough:
+ reasoning_effort: "none"
+
prompts:
- file://./dispatch-prompt.ts
@@ -218,20 +235,10 @@ tests:
- "documents per second"
- "documents/second"
- # Resume import (structured JSON via Output.object). These rows override the
- # provider to JSON mode; the shared markdown provider above can't be reused
- # because it would force JSON onto the review/match tasks too.
+ # Resume import (structured JSON via Output.object). JSON mode is set per-task
+ # by dispatch-prompt.ts, not per-provider, so these rows run against every
+ # provider above without forcing JSON onto the review/match tasks.
- description: "[resume-import] well-structured resume"
- provider:
- id: openai:chat:deepseek-v4-flash
- config:
- apiBaseUrl: https://api.deepseek.com
- apiKeyEnvar: DEEPSEEK_API_KEY
- showThinking: false
- temperature: 0
- max_tokens: 8192
- response_format:
- type: json_object
vars:
task: resume-import
resumeText: file://./resume-import/fixtures/standard.txt
@@ -252,16 +259,6 @@ tests:
value: file://./resume-import/assertions.ts:assertStandardCurrentJob
- description: "[resume-import] embedded prompt injection + unrecognized sections"
- provider:
- id: openai:chat:deepseek-v4-flash
- config:
- apiBaseUrl: https://api.deepseek.com
- apiKeyEnvar: DEEPSEEK_API_KEY
- showThinking: false
- temperature: 0
- max_tokens: 8192
- response_format:
- type: json_object
vars:
task: resume-import
resumeText: file://./resume-import/fixtures/adversarial-resume.txt
diff --git a/evals/resume-import/promptfooconfig.yaml b/evals/resume-import/promptfooconfig.yaml
index 1b5ae17..03c9a96 100644
--- a/evals/resume-import/promptfooconfig.yaml
+++ b/evals/resume-import/promptfooconfig.yaml
@@ -2,6 +2,7 @@ description: Resume import prompt regression tests
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -11,6 +12,21 @@ providers:
response_format:
type: json_object
+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in
+ # passthrough to clear promptfoo's reasoning-model name check; disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 8192
+ passthrough:
+ reasoning_effort: "none"
+ response_format:
+ type: json_object
+
prompts:
- file://./prompt.ts
diff --git a/evals/resume-review/promptfooconfig.yaml b/evals/resume-review/promptfooconfig.yaml
index 844e788..14ff2aa 100644
--- a/evals/resume-review/promptfooconfig.yaml
+++ b/evals/resume-review/promptfooconfig.yaml
@@ -2,6 +2,7 @@ description: Resume review prompt regression tests
providers:
- id: openai:chat:deepseek-v4-flash
+ label: deepseek-v4-flash
config:
apiBaseUrl: https://api.deepseek.com
apiKeyEnvar: DEEPSEEK_API_KEY
@@ -9,6 +10,19 @@ providers:
temperature: 0
max_tokens: 8192
+ # Local Ollama via its OpenAI-compatible endpoint. reasoning_effort goes in
+ # passthrough to clear promptfoo's reasoning-model name check; disabling
+ # thinking matches the app, where ollama-ai-provider-v2 sets think:false.
+ - id: openai:chat:qwen3.5:9b
+ label: ollama-qwen3.5-9b
+ config:
+ apiBaseUrl: http://localhost:11434/v1
+ apiKey: ollama
+ temperature: 0
+ max_tokens: 8192
+ passthrough:
+ reasoning_effort: "none"
+
prompts:
- file://./prompt.ts