Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

feat: Implement job deduplication logic and URL normalization in scraper utilities

Khuram Niazdde8518e4bde
diff --git a/__tests__/greenhouse-runner.spec.ts b/__tests__/greenhouse-runner.spec.ts
index 3023464..88c5b46 100644
--- a/__tests__/greenhouse-runner.spec.ts
+++ b/__tests__/greenhouse-runner.spec.ts
@@ -196,6 +196,50 @@ describe("runAutomation (greenhouse)", () => {
expect((generateText as any).mock.calls.length).toBe(0);
});
+ // Exercises getExistingJobKeys' wiring of real DB row shape (JobTitle/
+ // Company/Location relations) into jobDedupeKey, not just the pure
+ // dedupeJobs unit tested in scraper-utils.spec.ts.
+ it("dedupes against an existing DB job whose URL differs only by www/trailing-slash/tracking-param", async () => {
+ (prisma.job.findMany as any).mockResolvedValueOnce([
+ {
+ jobUrl: "https://www.job-boards.greenhouse.io/acme/jobs/1/?gh_src=abc",
+ JobTitle: { label: "Frontend Engineer" },
+ Company: { label: "Acme" },
+ Location: { label: "Remote" },
+ },
+ ]);
+
+ (searchGreenhouseJobs as any).mockResolvedValue({
+ jobs: [
+ {
+ ...makeJob("Frontend Engineer", "React"),
+ url: "https://job-boards.greenhouse.io/acme/jobs/1",
+ },
+ makeJob("Senior Frontend Engineer", "Vue"),
+ ],
+ errors: [],
+ });
+
+ const result = await runAutomation(automation);
+
+ expect(result.status).toBe("completed");
+ expect(result.jobsSearched).toBe(2);
+ // The first job matches the existing DB row's normalized key, so only
+ // the second (genuinely new) job is saved.
+ expect(result.jobsSaved).toBe(1);
+ expect((prisma.job.create as any).mock.calls).toHaveLength(1);
+
+ const updateArg = (prisma.automationRun.update as any).mock.calls.find(
+ (c: any[]) => c[0]?.where?.id === "run1",
+ );
+ const funnelStats = JSON.parse(updateArg[0].data.funnelStats);
+ const byKey = Object.fromEntries(
+ funnelStats.map((s: any) => [s.key, s.count]),
+ );
+ expect(byKey.fetched).toBe(2);
+ expect(byKey.dedup).toBe(1);
+ });
+
// Concurrency semantics (bounded p-limit dispatch, provider-gated).
// Ollama forces concurrency 1 (see beforeEach's default userSettings mock
// of null -> defaultUserSettings.ai, which is Ollama), so these tests
diff --git a/__tests__/scraper-utils.spec.ts b/__tests__/scraper-utils.spec.ts
index ef21243..82a7f7e 100644
--- a/__tests__/scraper-utils.spec.ts
+++ b/__tests__/scraper-utils.spec.ts
@@ -3,6 +3,8 @@ import {
normalizeForSearch,
extractKeywords,
extractCityName,
+ jobDedupeKey,
+ dedupeJobs,
} from "@/lib/scraper/utils";
describe("normalizeJobUrl", () => {
@@ -65,6 +67,96 @@ describe("normalizeJobUrl", () => {
const url = "https://example.com/job/123";
expect(normalizeJobUrl(url)).toBe("https://example.com/job/123");
});
+
+ it("removes the greenhouse gh_src tracking param but keeps gh_jid", () => {
+ const url =
+ "https://boards.greenhouse.io/acme/jobs/123?gh_src=abc&gh_jid=123";
+ expect(normalizeJobUrl(url)).toBe(
+ "https://boards.greenhouse.io/acme/jobs/123?gh_jid=123"
+ );
+ });
+
+ it("strips a trailing slash from the path", () => {
+ expect(normalizeJobUrl("https://example.com/jobs/123/")).toBe(
+ "https://example.com/jobs/123"
+ );
+ });
+
+ it("sorts query params so order does not matter", () => {
+ expect(normalizeJobUrl("https://example.com/job?b=2&a=1")).toBe(
+ "https://example.com/job?a=1&b=2"
+ );
+ });
+});
+
+describe("jobDedupeKey", () => {
+ it("treats trailing slash, host case, and www as the same job", () => {
+ const a = jobDedupeKey({ url: "https://www.example.com/jobs/1/" });
+ const b = jobDedupeKey({ url: "https://Example.com/jobs/1" });
+ expect(a).toBe(b);
+ });
+
+ it("treats differing tracking params as the same job", () => {
+ const a = jobDedupeKey({ url: "https://ex.com/j/1?gh_src=x&gh_jid=1" });
+ const b = jobDedupeKey({ url: "https://ex.com/j/1?gh_src=y&gh_jid=1" });
+ expect(a).toBe(b);
+ });
+
+ it("falls back to a title/company/location key when url is missing", () => {
+ const a = jobDedupeKey({
+ title: "Software Engineer",
+ company: "Acme",
+ location: "Remote",
+ });
+ const b = jobDedupeKey({
+ title: "Software Engineer",
+ company: "Acme",
+ location: "Remote",
+ });
+ expect(a).toBe(b);
+ expect(a.startsWith("meta:")).toBe(true);
+ });
+
+ it("distinguishes different linkless jobs", () => {
+ const a = jobDedupeKey({ title: "Engineer", company: "Acme" });
+ const b = jobDedupeKey({ title: "Designer", company: "Acme" });
+ expect(a).not.toBe(b);
+ });
+});
+
+describe("dedupeJobs", () => {
+ it("removes duplicates within the batch", () => {
+ const jobs = [
+ { title: "A", company: "X", location: "R", url: "https://ex.com/1" },
+ { title: "A", company: "X", location: "R", url: "https://ex.com/1?ref=y" },
+ ];
+ expect(dedupeJobs(jobs, new Set())).toHaveLength(1);
+ });
+
+ it("removes jobs already saved (existing keys)", () => {
+ const jobs = [
+ { title: "A", company: "X", location: "R", url: "https://ex.com/1" },
+ ];
+ const existing = new Set([jobDedupeKey(jobs[0])]);
+ expect(dedupeJobs(jobs, existing)).toHaveLength(0);
+ });
+
+ it("dedups linkless jobs by metadata", () => {
+ const jobs = [
+ { title: "A", company: "X", location: "R", url: "" },
+ { title: "A", company: "X", location: "R", url: "" },
+ { title: "B", company: "X", location: "R", url: "" },
+ ];
+ expect(dedupeJobs(jobs, new Set())).toHaveLength(2);
+ });
+
+ it("keeps genuinely distinct jobs", () => {
+ const jobs = [
+ { title: "A", company: "X", location: "R", url: "https://ex.com/1" },
+ { title: "B", company: "Y", location: "R", url: "https://ex.com/2" },
+ ];
+ expect(dedupeJobs(jobs, new Set())).toHaveLength(2);
+ });
});
describe("normalizeForSearch", () => {
diff --git a/src/lib/scraper/runner.ts b/src/lib/scraper/runner.ts
index 5a99983..6f41078 100644
--- a/src/lib/scraper/runner.ts
+++ b/src/lib/scraper/runner.ts
@@ -14,7 +14,7 @@ import { searchGreenhouseJobs } from "./greenhouse";
import { runGreenhousePipeline } from "./greenhouse/pipeline";
import type { ScoredJob } from "./greenhouse/pipeline";
import { mapScrapedJobToJobRecord } from "./mapper";
-import { normalizeJobUrl } from "./utils";
+import { normalizeJobUrl, dedupeJobs, jobDedupeKey } from "./utils";
import { calculateNextRunAt } from "./schedule";
import { APP_CONSTANTS } from "@/lib/constants";
import {
@@ -373,10 +373,8 @@ export async function runAutomation(
"Checking for duplicate jobs...",
);
- const existingJobUrls = await getExistingJobUrls(automation.userId);
- const newJobs = searchResult.data.filter(
- (job) => !existingJobUrls.has(normalizeJobUrl(job.url)),
- );
+ const existingKeys = await getExistingJobKeys(automation.userId);
+ const newJobs = dedupeJobs(searchResult.data, existingKeys);
const jobsDeduplicated = newJobs.length;
automationLogger.log(
@@ -608,19 +606,29 @@ export async function runAutomation(
}
}
-async function getExistingJobUrls(userId: string): Promise<Set<string>> {
+async function getExistingJobKeys(userId: string): Promise<Set<string>> {
const existingJobs = await db.job.findMany({
where: { userId },
- select: { jobUrl: true },
+ select: {
+ jobUrl: true,
+ JobTitle: { select: { label: true } },
+ Company: { select: { label: true } },
+ Location: { select: { label: true } },
+ },
});
- const urls = new Set<string>();
+ const keys = new Set<string>();
for (const job of existingJobs) {
- if (job.jobUrl) {
- urls.add(normalizeJobUrl(job.jobUrl));
- }
+ keys.add(
+ jobDedupeKey({
+ url: job.jobUrl,
+ title: job.JobTitle?.label,
+ company: job.Company?.label,
+ location: job.Location?.label ?? undefined,
+ }),
+ );
}
- return urls;
+ return keys;
}
interface GreenhouseRunConfig {
@@ -753,15 +761,8 @@ async function runGreenhouseRun(
);
// Dedup against existing jobs and within this batch.
- const existingJobUrls = await getExistingJobUrls(automation.userId);
- const seen = new Set<string>();
- const dedupedJobs: JobDetails[] = [];
- for (const job of jobs) {
- const normalized = normalizeJobUrl(job.url);
- if (existingJobUrls.has(normalized) || seen.has(normalized)) continue;
- seen.add(normalized);
- dedupedJobs.push(job);
- }
+ const existingKeys = await getExistingJobKeys(automation.userId);
+ const dedupedJobs = dedupeJobs(jobs, existingKeys);
const jobsDeduplicated = dedupedJobs.length;
automationLogger.log(
diff --git a/src/lib/scraper/utils.ts b/src/lib/scraper/utils.ts
index 51b0ae4..67a9ad2 100644
--- a/src/lib/scraper/utils.ts
+++ b/src/lib/scraper/utils.ts
@@ -12,20 +12,78 @@ const TRACKING_PARAMS = [
"tk",
"from",
"vjk",
+ "gh_src",
];
+// Normalizes a URL for storage/clicking: only touches things that never change
+// which resource the URL points to (tracking params, fragment, param order,
+// trailing slash). Deliberately keeps host/protocol/www intact so the stored
+// link stays valid.
export function normalizeJobUrl(url: string): string {
try {
const parsed = new URL(url);
+ parsed.hash = "";
TRACKING_PARAMS.forEach((param) => {
parsed.searchParams.delete(param);
});
+ parsed.searchParams.sort();
+ if (parsed.pathname.length > 1) {
+ parsed.pathname = parsed.pathname.replace(/\/+$/, "");
+ }
return parsed.toString();
} catch {
return url;
}
}
+// Aggressive canonical form used only as a dedup comparison key (never stored
+// or clicked), so it can safely fold host case and a leading "www.".
+function urlDedupeKey(url: string): string {
+ try {
+ const parsed = new URL(normalizeJobUrl(url));
+ const host = parsed.hostname.toLowerCase().replace(/^www\./, "");
+ return `${host}${parsed.pathname}${parsed.search}`;
+ } catch {
+ return url.trim().toLowerCase();
+ }
+}
+
+interface DedupableJob {
+ url?: string | null;
+ title?: string;
+ company?: string;
+ location?: string;
+}
+
+// One key per job, shared by both the incoming batch and existing DB records so
+// they compare identically. Falls back to a title/company/location signature
+// when a job has no usable URL (otherwise linkless jobs re-add every run).
+export function jobDedupeKey(job: DedupableJob): string {
+ const url = job.url?.trim();
+ if (url) return `url:${urlDedupeKey(url)}`;
+ const meta = [job.title, job.company, job.location]
+ .map((part) => normalizeForSearch(part ?? ""))
+ .join("|");
+ return `meta:${meta}`;
+}
+
+// Removes jobs already saved (existingKeys) and collapses duplicates within the
+// batch itself. Both source paths (JSearch, Greenhouse) run through here.
+export function dedupeJobs<T extends DedupableJob>(
+ jobs: T[],
+ existingKeys: Set<string>,
+): T[] {
+ const seen = new Set<string>();
+ const result: T[] = [];
+ for (const job of jobs) {
+ const key = jobDedupeKey(job);
+ if (existingKeys.has(key) || seen.has(key)) continue;
+ seen.add(key);
+ result.push(job);
+ }
+ return result;
+}
+
export function normalizeForSearch(str: string): string {
return str
.toLowerCase()