Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
fix(automation): make the match threshold a save gate, add a minimum prerank score, and update the wizard copy, wiki and specs
Khuram Niaz82e03c2b2aa1
diff --git a/__tests__/ats-pipeline.spec.ts b/__tests__/ats-pipeline.spec.tsindex fdcb100..ab20089 100644--- a/__tests__/ats-pipeline.spec.ts+++ b/__tests__/ats-pipeline.spec.ts@@ -1,4 +1,5 @@import { runAtsPipeline } from "@/lib/scraper/ats/pipeline";+import { APP_CONSTANTS } from "@/lib/constants";import type { JobDetails } from "@/lib/scraper/types";function job(overrides: Partial<JobDetails>): JobDetails {@@ -26,12 +27,55 @@ describe("runAtsPipeline", () => {job({ title: "Cook", description: "make food" }), // no signaljob({ title: "Chef", description: "react typescript daily" }), // 2 kw hits];- const result = runAtsPipeline(jobs, config, []);+ // Widen the corpus so the user's terms read as rare and both hitters clear+ // the minimum-score gate — this test is about the floor, not that gate.+ const corpus = [+ ...jobs,+ ...Array.from({ length: 10 }, () =>+ job({ title: "Cook", description: "make food" }),+ ),+ ];+ const result = runAtsPipeline(jobs, config, [], { corpus });expect(result.funnel.relevant).toBe(2);const all = [...result.toAnalyze, ...result.toSaveUnanalyzed];expect(all).toHaveLength(2);});+ it("cuts floor-passers that score below the minimum", () => {+ const jobs = [+ job({ title: "Frontend Engineer", description: "react typescript" }),+ job({ title: "Chef", description: "react once" }), // lone generic hit+ ];+ const corpus = [+ ...jobs,+ // "react" is everywhere in this corpus, so it carries almost no weight;+ // "typescript" and the title tokens stay rare.+ ...Array.from({ length: 10 }, () =>+ job({ title: "Cook", description: "react" }),+ ),+ ];+ const result = runAtsPipeline(jobs, config, [], { corpus });+ expect(result.funnel.floorSurvivors).toBe(2);+ expect(result.funnel.scoreCut).toBe(1);+ expect(result.funnel.relevant).toBe(1);+ expect(result.toAnalyze[0].job.title).toBe("Frontend Engineer");+ });++ it("fails open when the score gate would drop every floor-passer", () => {+ // Every job shares every search term, so idf drives all scores to ~0.+ const jobs = Array.from({ length: 4 }, () =>+ job({ title: "Frontend Engineer", description: "react typescript" }),+ );+ const result = runAtsPipeline(jobs, config, []);+ // Premise: without the fail-open these would all have been cut.+ for (const s of result.toAnalyze) {+ expect(s.score).toBeLessThan(APP_CONSTANTS.ATS_MIN_PRERANK_SCORE);+ }+ expect(result.funnel.floorSurvivors).toBe(4);+ expect(result.funnel.scoreCut).toBe(0);+ expect(result.funnel.relevant).toBe(4);+ });+it("caps toAnalyze at K and pushes the rest to un-analyzed", () => {const jobs = Array.from({ length: 8 }, (_, i) =>job({ title: "Frontend Engineer", description: `React ${i}` }),diff --git a/__tests__/ats-runner.spec.ts b/__tests__/ats-runner.spec.tsindex f4b2f05..16dc216 100644--- a/__tests__/ats-runner.spec.ts+++ b/__tests__/ats-runner.spec.ts@@ -142,6 +142,24 @@ describe("runAutomation (lever)", () => {expect(result.jobsSaved).toBe(1);});+ it("does not save an analyzed job scoring below the match threshold", async () => {+ (searchLeverJobs as any).mockResolvedValue({+ jobs: [makeJob("Frontend Engineer", "React")],+ errors: [],+ });+ (generateText as any).mockResolvedValue({+ text: "SCORES: match=40 recommendation=weak match\n\n## Summary\nThin fit",+ });++ const result = await runAutomation(leverAutomation);++ expect(result.status).toBe("completed");+ expect(result.jobsProcessed).toBe(1); // analyzed+ expect(result.jobsMatched).toBe(0);+ expect(result.jobsSaved).toBe(0);+ expect((prisma.job.create as any).mock.calls.length).toBe(0);+ });+it("persists Lever's workplaceType through to the job record", async () => {(searchLeverJobs as any).mockResolvedValue({jobs: [makeJob("Frontend Engineer", "React", { workplaceType: "HYBRID" })],diff --git a/__tests__/greenhouse-runner.spec.ts b/__tests__/greenhouse-runner.spec.tsindex 0b40bbe..fa80de3 100644--- a/__tests__/greenhouse-runner.spec.ts+++ b/__tests__/greenhouse-runner.spec.ts@@ -419,8 +419,8 @@ describe("runAutomation (greenhouse)", () => {expect(result.status).toBe("completed");expect((generateText as any).mock.calls.length).toBe(5);expect(result.jobsProcessed).toBe(5); // analyzed- expect(result.jobsMatched).toBe(3); // highlighted: 90, 85, 95- expect(result.jobsSaved).toBe(5);+ expect(result.jobsMatched).toBe(3); // met threshold: 90, 85, 95+ expect(result.jobsSaved).toBe(3); // 50 and 60 are analyzed but dropped});});});diff --git a/src/components/automations/automation-wizard/StepMatching.tsx b/src/components/automations/automation-wizard/StepMatching.tsxindex bba44ee..964703e 100644--- a/src/components/automations/automation-wizard/StepMatching.tsx+++ b/src/components/automations/automation-wizard/StepMatching.tsx@@ -34,9 +34,9 @@ export function StepMatching({/></FormControl><FormDescription>- Highlight listings whose AI match score exceeds this threshold.- Relevant listings are saved regardless — this only flags strong- matches.+ Only save listings whose AI match score reaches this threshold.+ Analyzed listings that score below it are discarded — set it lower+ to keep more.</FormDescription><FormMessage /></FormItem>diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 145b69e..6f1978e 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -106,6 +106,9 @@ export const APP_CONSTANTS = {ATS_LISTING_CAP: 50, // safety ceiling applied after the relevance floorATS_FLOOR_MIN_TITLE_HITS: 1,ATS_FLOOR_MIN_KEYWORD_HITS: 1,+ // Minimum weighted prerank score to spend an LLM call on. Measured against+ // real runs: nothing scoring below this ever exceeded a 6% AI match.+ ATS_MIN_PRERANK_SCORE: 0.1,ATS_TITLE_WEIGHT: 0.6,ATS_SKILL_WEIGHT: 0.4,diff --git a/src/lib/scraper/ats/pipeline.ts b/src/lib/scraper/ats/pipeline.tsindex bae32eb..ef17670 100644--- a/src/lib/scraper/ats/pipeline.ts+++ b/src/lib/scraper/ats/pipeline.ts@@ -25,7 +25,8 @@ export interface PipelineResult {deduped: number; // jobs handed in (already deduped by the runner)located: number | null; // survivors after strict location gate (null if off)floorSurvivors: number; // jobs clearing the floor, before the cap ceiling- relevant: number; // floor survivors after the cap ceiling (== total saved)+ scoreCut: number; // floor survivors dropped by the minimum-score gate+ relevant: number; // survivors after the score gate and cap (the LLM budget)};}@@ -71,7 +72,17 @@ export function runAtsPipeline(.filter((s) => passesFloor(s.components)).sort((a, b) => b.score - a.score);- const capped = floorSurvivors.slice(0, cap);+ // Term presence is not enough to be worth an LLM call — a single generic hit+ // clears the floor. Cut the weak tail by weighted score too. Fails open: idf+ // is corpus-relative, so a board where every job shares the user's terms+ // scores everything near zero, and dropping that whole run would be worse+ // than analyzing it.+ const strong = floorSurvivors.filter(+ (s) => s.score >= APP_CONSTANTS.ATS_MIN_PRERANK_SCORE,+ );+ const ranked = strong.length > 0 ? strong : floorSurvivors;++ const capped = ranked.slice(0, cap);return {toAnalyze: capped.slice(0, k),@@ -80,6 +91,7 @@ export function runAtsPipeline(deduped,located: located ? located.length : null,floorSurvivors: floorSurvivors.length,+ scoreCut: floorSurvivors.length - ranked.length,relevant: capped.length,},};diff --git a/src/lib/scraper/automation-run/atsRun.ts b/src/lib/scraper/automation-run/atsRun.tsindex fec0541..e7262a4 100644--- a/src/lib/scraper/automation-run/atsRun.ts+++ b/src/lib/scraper/automation-run/atsRun.ts@@ -140,6 +140,13 @@ export async function runAtsRun("info",`${label} ${pipeline.funnel.floorSurvivors} jobs cleared the relevance floor${capped}`,);+ if (pipeline.funnel.scoreCut > 0) {+ automationLogger.log(+ automation.id,+ "info",+ `${label} ${pipeline.funnel.scoreCut} of them too weakly related to analyze — skipped`,+ );+ }const buildFunnel = (analyzed: number, highlighted: number): string => {const stages: FunnelStage[] = [@@ -209,6 +216,7 @@ export async function runAtsRun(let jobsSaved = 0;let analyzed = 0;let highlighted = 0;+ let belowThreshold = 0;let aiError: string | null = null;// Save the un-analyzed tier (floor survivors beyond the top-K).@@ -326,14 +334,17 @@ export async function runAtsRun(analyzed++;const isStrong = matchResult.score >= automation.matchThreshold;if (isStrong) highlighted++;+ else belowThreshold++;automationLogger.log(automation.id,isStrong ? "success" : "info",- `${label} Analyzed ${analyzed}/${totalToAnalyze}: ${scored.job.title} — ${matchResult.score}%`,+ `${label} Analyzed ${analyzed}/${totalToAnalyze}: ${scored.job.title} — ${matchResult.score}%${isStrong ? "" : ` (below ${automation.matchThreshold}% threshold — not saved)`}`,{ score: matchResult.score, threshold: automation.matchThreshold },);+ if (!isStrong) return;+try {const saved = await persistDiscoveredJob(automation,@@ -379,6 +390,14 @@ export async function runAtsRun(`${label} LLM analysis complete (${analyzed}/${pipeline.toAnalyze.length} succeeded)`,);+ if (belowThreshold > 0) {+ automationLogger.log(+ automation.id,+ "info",+ `${label} ${belowThreshold} analyzed job(s) scored below your ${automation.matchThreshold}% threshold and were not saved`,+ );+ }+automationLogger.endRun(automation.id);return await finalizeRun(runId, {diff --git a/wiki/automations.md b/wiki/automations.mdindex f13b65a..50728d5 100644--- a/wiki/automations.md+++ b/wiki/automations.md@@ -26,7 +26,7 @@ Open **Automations** in the sidebar and click **Create Automation**. The wizard1. **Basics** — a name, and which job board (Greenhouse, Lever or Ashby) the companies live on.2. **Search** — the companies to track, then your filters: target job titles, keywords/skills, locations, and how many listings get full AI analysis per run. This is the step that decides how useful the automation is; see the next two sections.3. **Resume** — the resume every discovered job is scored against.-4. **Matching** — a match threshold. Listings scoring above it are flagged as strong matches; nothing is thrown away because of it.+4. **Matching** — a match threshold. Only listings whose AI match score reaches it are saved; anything analyzed and scored below it is discarded.5. **Schedule** — the hour of the day the automation runs (server time). Only one automation may occupy an hour, so hours already taken are marked **In use** and rejected.6. **Review** — confirm and save.@@ -72,6 +72,8 @@ Two tiers. **Jobs analyzed per run** (the slider in the Search step, default 10,**Save additional relevant listings** is on by default. With it on, relevant listings that did not make the cut are still saved, ranked but not yet AI-scored — they show a lexical relevance percentage and an **Analyze** button so you can score one on demand. Turn it off if you only want the scored shortlist.+Listings only reach AI analysis if they are related to your search strongly enough to be worth the call — clearing the relevance floor on a single generic word is not enough. Of the ones that are analyzed, only those meeting your match threshold are saved. So a run can fetch hundreds of listings and save none: the run log shows how many were skipped as too weakly related and how many were analyzed but fell below the threshold.+## When does it run, and can I run one now?An automation runs daily at the hour you chose, in the server's timezone, as long as its status is **active**. Open an automation to run it immediately with **Run Now** — manual runs are limited to **5 per hour**. A run in progress can be stopped with **Abort Run**.