Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
feat: Implement job deduplication logic and URL normalization in scraper utilities
Khuram Niazdde8518e4bde
diff --git a/__tests__/greenhouse-runner.spec.ts b/__tests__/greenhouse-runner.spec.tsindex 3023464..88c5b46 100644--- a/__tests__/greenhouse-runner.spec.ts+++ b/__tests__/greenhouse-runner.spec.ts@@ -196,6 +196,50 @@ describe("runAutomation (greenhouse)", () => {expect((generateText as any).mock.calls.length).toBe(0);});+ // Exercises getExistingJobKeys' wiring of real DB row shape (JobTitle/+ // Company/Location relations) into jobDedupeKey, not just the pure+ // dedupeJobs unit tested in scraper-utils.spec.ts.+ it("dedupes against an existing DB job whose URL differs only by www/trailing-slash/tracking-param", async () => {+ (prisma.job.findMany as any).mockResolvedValueOnce([+ {+ jobUrl: "https://www.job-boards.greenhouse.io/acme/jobs/1/?gh_src=abc",+ JobTitle: { label: "Frontend Engineer" },+ Company: { label: "Acme" },+ Location: { label: "Remote" },+ },+ ]);++ (searchGreenhouseJobs as any).mockResolvedValue({+ jobs: [+ {+ ...makeJob("Frontend Engineer", "React"),+ url: "https://job-boards.greenhouse.io/acme/jobs/1",+ },+ makeJob("Senior Frontend Engineer", "Vue"),+ ],+ errors: [],+ });++ const result = await runAutomation(automation);++ expect(result.status).toBe("completed");+ expect(result.jobsSearched).toBe(2);+ // The first job matches the existing DB row's normalized key, so only+ // the second (genuinely new) job is saved.+ expect(result.jobsSaved).toBe(1);+ expect((prisma.job.create as any).mock.calls).toHaveLength(1);++ const updateArg = (prisma.automationRun.update as any).mock.calls.find(+ (c: any[]) => c[0]?.where?.id === "run1",+ );+ const funnelStats = JSON.parse(updateArg[0].data.funnelStats);+ const byKey = Object.fromEntries(+ funnelStats.map((s: any) => [s.key, s.count]),+ );+ expect(byKey.fetched).toBe(2);+ expect(byKey.dedup).toBe(1);+ });+// Concurrency semantics (bounded p-limit dispatch, provider-gated).// Ollama forces concurrency 1 (see beforeEach's default userSettings mock// of null -> defaultUserSettings.ai, which is Ollama), so these testsdiff --git a/__tests__/scraper-utils.spec.ts b/__tests__/scraper-utils.spec.tsindex ef21243..82a7f7e 100644--- a/__tests__/scraper-utils.spec.ts+++ b/__tests__/scraper-utils.spec.ts@@ -3,6 +3,8 @@ import {normalizeForSearch,extractKeywords,extractCityName,+ jobDedupeKey,+ dedupeJobs,} from "@/lib/scraper/utils";describe("normalizeJobUrl", () => {@@ -65,6 +67,96 @@ describe("normalizeJobUrl", () => {const url = "https://example.com/job/123";expect(normalizeJobUrl(url)).toBe("https://example.com/job/123");});++ it("removes the greenhouse gh_src tracking param but keeps gh_jid", () => {+ const url =+ "https://boards.greenhouse.io/acme/jobs/123?gh_src=abc&gh_jid=123";+ expect(normalizeJobUrl(url)).toBe(+ "https://boards.greenhouse.io/acme/jobs/123?gh_jid=123"+ );+ });++ it("strips a trailing slash from the path", () => {+ expect(normalizeJobUrl("https://example.com/jobs/123/")).toBe(+ "https://example.com/jobs/123"+ );+ });++ it("sorts query params so order does not matter", () => {+ expect(normalizeJobUrl("https://example.com/job?b=2&a=1")).toBe(+ "https://example.com/job?a=1&b=2"+ );+ });+});++describe("jobDedupeKey", () => {+ it("treats trailing slash, host case, and www as the same job", () => {+ const a = jobDedupeKey({ url: "https://www.example.com/jobs/1/" });+ const b = jobDedupeKey({ url: "https://Example.com/jobs/1" });+ expect(a).toBe(b);+ });++ it("treats differing tracking params as the same job", () => {+ const a = jobDedupeKey({ url: "https://ex.com/j/1?gh_src=x&gh_jid=1" });+ const b = jobDedupeKey({ url: "https://ex.com/j/1?gh_src=y&gh_jid=1" });+ expect(a).toBe(b);+ });++ it("falls back to a title/company/location key when url is missing", () => {+ const a = jobDedupeKey({+ title: "Software Engineer",+ company: "Acme",+ location: "Remote",+ });+ const b = jobDedupeKey({+ title: "Software Engineer",+ company: "Acme",+ location: "Remote",+ });+ expect(a).toBe(b);+ expect(a.startsWith("meta:")).toBe(true);+ });++ it("distinguishes different linkless jobs", () => {+ const a = jobDedupeKey({ title: "Engineer", company: "Acme" });+ const b = jobDedupeKey({ title: "Designer", company: "Acme" });+ expect(a).not.toBe(b);+ });+});++describe("dedupeJobs", () => {+ it("removes duplicates within the batch", () => {+ const jobs = [+ { title: "A", company: "X", location: "R", url: "https://ex.com/1" },+ { title: "A", company: "X", location: "R", url: "https://ex.com/1?ref=y" },+ ];+ expect(dedupeJobs(jobs, new Set())).toHaveLength(1);+ });++ it("removes jobs already saved (existing keys)", () => {+ const jobs = [+ { title: "A", company: "X", location: "R", url: "https://ex.com/1" },+ ];+ const existing = new Set([jobDedupeKey(jobs[0])]);+ expect(dedupeJobs(jobs, existing)).toHaveLength(0);+ });++ it("dedups linkless jobs by metadata", () => {+ const jobs = [+ { title: "A", company: "X", location: "R", url: "" },+ { title: "A", company: "X", location: "R", url: "" },+ { title: "B", company: "X", location: "R", url: "" },+ ];+ expect(dedupeJobs(jobs, new Set())).toHaveLength(2);+ });++ it("keeps genuinely distinct jobs", () => {+ const jobs = [+ { title: "A", company: "X", location: "R", url: "https://ex.com/1" },+ { title: "B", company: "Y", location: "R", url: "https://ex.com/2" },+ ];+ expect(dedupeJobs(jobs, new Set())).toHaveLength(2);+ });});describe("normalizeForSearch", () => {diff --git a/src/lib/scraper/runner.ts b/src/lib/scraper/runner.tsindex 5a99983..6f41078 100644--- a/src/lib/scraper/runner.ts+++ b/src/lib/scraper/runner.ts@@ -14,7 +14,7 @@ import { searchGreenhouseJobs } from "./greenhouse";import { runGreenhousePipeline } from "./greenhouse/pipeline";import type { ScoredJob } from "./greenhouse/pipeline";import { mapScrapedJobToJobRecord } from "./mapper";-import { normalizeJobUrl } from "./utils";+import { normalizeJobUrl, dedupeJobs, jobDedupeKey } from "./utils";import { calculateNextRunAt } from "./schedule";import { APP_CONSTANTS } from "@/lib/constants";import {@@ -373,10 +373,8 @@ export async function runAutomation("Checking for duplicate jobs...",);- const existingJobUrls = await getExistingJobUrls(automation.userId);- const newJobs = searchResult.data.filter(- (job) => !existingJobUrls.has(normalizeJobUrl(job.url)),- );+ const existingKeys = await getExistingJobKeys(automation.userId);+ const newJobs = dedupeJobs(searchResult.data, existingKeys);const jobsDeduplicated = newJobs.length;automationLogger.log(@@ -608,19 +606,29 @@ export async function runAutomation(}}-async function getExistingJobUrls(userId: string): Promise<Set<string>> {+async function getExistingJobKeys(userId: string): Promise<Set<string>> {const existingJobs = await db.job.findMany({where: { userId },- select: { jobUrl: true },+ select: {+ jobUrl: true,+ JobTitle: { select: { label: true } },+ Company: { select: { label: true } },+ Location: { select: { label: true } },+ },});- const urls = new Set<string>();+ const keys = new Set<string>();for (const job of existingJobs) {- if (job.jobUrl) {- urls.add(normalizeJobUrl(job.jobUrl));- }+ keys.add(+ jobDedupeKey({+ url: job.jobUrl,+ title: job.JobTitle?.label,+ company: job.Company?.label,+ location: job.Location?.label ?? undefined,+ }),+ );}- return urls;+ return keys;}interface GreenhouseRunConfig {@@ -753,15 +761,8 @@ async function runGreenhouseRun();// Dedup against existing jobs and within this batch.- const existingJobUrls = await getExistingJobUrls(automation.userId);- const seen = new Set<string>();- const dedupedJobs: JobDetails[] = [];- for (const job of jobs) {- const normalized = normalizeJobUrl(job.url);- if (existingJobUrls.has(normalized) || seen.has(normalized)) continue;- seen.add(normalized);- dedupedJobs.push(job);- }+ const existingKeys = await getExistingJobKeys(automation.userId);+ const dedupedJobs = dedupeJobs(jobs, existingKeys);const jobsDeduplicated = dedupedJobs.length;automationLogger.log(diff --git a/src/lib/scraper/utils.ts b/src/lib/scraper/utils.tsindex 51b0ae4..67a9ad2 100644--- a/src/lib/scraper/utils.ts+++ b/src/lib/scraper/utils.ts@@ -12,20 +12,78 @@ const TRACKING_PARAMS = ["tk","from","vjk",+ "gh_src",];+// Normalizes a URL for storage/clicking: only touches things that never change+// which resource the URL points to (tracking params, fragment, param order,+// trailing slash). Deliberately keeps host/protocol/www intact so the stored+// link stays valid.export function normalizeJobUrl(url: string): string {try {const parsed = new URL(url);+ parsed.hash = "";TRACKING_PARAMS.forEach((param) => {parsed.searchParams.delete(param);});+ parsed.searchParams.sort();+ if (parsed.pathname.length > 1) {+ parsed.pathname = parsed.pathname.replace(/\/+$/, "");+ }return parsed.toString();} catch {return url;}}+// Aggressive canonical form used only as a dedup comparison key (never stored+// or clicked), so it can safely fold host case and a leading "www.".+function urlDedupeKey(url: string): string {+ try {+ const parsed = new URL(normalizeJobUrl(url));+ const host = parsed.hostname.toLowerCase().replace(/^www\./, "");+ return `${host}${parsed.pathname}${parsed.search}`;+ } catch {+ return url.trim().toLowerCase();+ }+}++interface DedupableJob {+ url?: string | null;+ title?: string;+ company?: string;+ location?: string;+}++// One key per job, shared by both the incoming batch and existing DB records so+// they compare identically. Falls back to a title/company/location signature+// when a job has no usable URL (otherwise linkless jobs re-add every run).+export function jobDedupeKey(job: DedupableJob): string {+ const url = job.url?.trim();+ if (url) return `url:${urlDedupeKey(url)}`;+ const meta = [job.title, job.company, job.location]+ .map((part) => normalizeForSearch(part ?? ""))+ .join("|");+ return `meta:${meta}`;+}++// Removes jobs already saved (existingKeys) and collapses duplicates within the+// batch itself. Both source paths (JSearch, Greenhouse) run through here.+export function dedupeJobs<T extends DedupableJob>(+ jobs: T[],+ existingKeys: Set<string>,+): T[] {+ const seen = new Set<string>();+ const result: T[] = [];+ for (const job of jobs) {+ const key = jobDedupeKey(job);+ if (existingKeys.has(key) || seen.has(key)) continue;+ seen.add(key);+ result.push(job);+ }+ return result;+}+export function normalizeForSearch(str: string): string {return str.toLowerCase()