Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

refactor(scraper): move the shared ranking pipeline, HTML helpers and ATS_* constants out of greenhouse/, updating components and specs

Khuram Niazb9494173ad4c
diff --git a/__tests__/greenhouse-pipeline.spec.ts b/__tests__/ats-pipeline.spec.ts
similarity index 84%
rename from __tests__/greenhouse-pipeline.spec.ts
rename to __tests__/ats-pipeline.spec.ts
index 68c8441..fdcb100 100644
--- a/__tests__/greenhouse-pipeline.spec.ts
+++ b/__tests__/ats-pipeline.spec.ts
@@ -1,4 +1,4 @@
-import { runGreenhousePipeline } from "@/lib/scraper/greenhouse/pipeline";
+import { runAtsPipeline } from "@/lib/scraper/ats/pipeline";
import type { JobDetails } from "@/lib/scraper/types";
function job(overrides: Partial<JobDetails>): JobDetails {
@@ -19,14 +19,14 @@ const config = {
strictLocation: false,
};
-describe("runGreenhousePipeline", () => {
+describe("runAtsPipeline", () => {
it("keeps only floor-passers", () => {
const jobs = [
job({ title: "Frontend Engineer", description: "React" }), // title hit
job({ title: "Cook", description: "make food" }), // no signal
job({ title: "Chef", description: "react typescript daily" }), // 2 kw hits
];
- const result = runGreenhousePipeline(jobs, config, []);
+ const result = runAtsPipeline(jobs, config, []);
expect(result.funnel.relevant).toBe(2);
const all = [...result.toAnalyze, ...result.toSaveUnanalyzed];
expect(all).toHaveLength(2);
@@ -36,7 +36,7 @@ describe("runGreenhousePipeline", () => {
const jobs = Array.from({ length: 8 }, (_, i) =>
job({ title: "Frontend Engineer", description: `React ${i}` }),
);
- const result = runGreenhousePipeline(jobs, config, [], { k: 3, cap: 50 });
+ const result = runAtsPipeline(jobs, config, [], { k: 3, cap: 50 });
expect(result.toAnalyze).toHaveLength(3);
expect(result.toSaveUnanalyzed).toHaveLength(5);
});
@@ -45,7 +45,7 @@ describe("runGreenhousePipeline", () => {
const jobs = Array.from({ length: 10 }, () =>
job({ title: "Frontend Engineer" }),
);
- const result = runGreenhousePipeline(jobs, config, [], { k: 2, cap: 4 });
+ const result = runAtsPipeline(jobs, config, [], { k: 2, cap: 4 });
expect(result.funnel.relevant).toBe(4);
// The uncapped count stays visible so the log can't report the cap as
// though it were the floor result.
@@ -56,7 +56,7 @@ describe("runGreenhousePipeline", () => {
});
it("omits the located stage when strictLocation is off", () => {
- const result = runGreenhousePipeline(
+ const result = runAtsPipeline(
[job({ title: "Frontend Engineer" })],
config,
[],
@@ -70,7 +70,7 @@ describe("runGreenhousePipeline", () => {
job({ title: "Frontend Engineer", location: "Berlin, Germany" }),
job({ title: "Frontend Engineer", location: "Remote" }),
];
- const result = runGreenhousePipeline(
+ const result = runAtsPipeline(
jobs,
{ ...config, strictLocation: true },
[],
@@ -87,7 +87,7 @@ describe("runGreenhousePipeline", () => {
location: "Canada",
}),
];
- const result = runGreenhousePipeline(jobs, config, [], { k: 2 });
+ const result = runAtsPipeline(jobs, config, [], { k: 2 });
expect(result.toAnalyze[0].score).toBeGreaterThanOrEqual(
result.toAnalyze[1].score,
);
diff --git a/__tests__/greenhouse-rank.spec.ts b/__tests__/ats-rank.spec.ts
similarity index 99%
rename from __tests__/greenhouse-rank.spec.ts
rename to __tests__/ats-rank.spec.ts
index bf66d7c..583637b 100644
--- a/__tests__/greenhouse-rank.spec.ts
+++ b/__tests__/ats-rank.spec.ts
@@ -3,7 +3,7 @@ import {
passesFloor,
locationMatches,
buildIdf,
-} from "@/lib/scraper/greenhouse/rank";
+} from "@/lib/scraper/ats/rank";
import type { JobDetails } from "@/lib/scraper/types";
function job(overrides: Partial<JobDetails>): JobDetails {
diff --git a/__tests__/greenhouse-provider.spec.ts b/__tests__/greenhouse-provider.spec.ts
index 1115192..3c1c1f8 100644
--- a/__tests__/greenhouse-provider.spec.ts
+++ b/__tests__/greenhouse-provider.spec.ts
@@ -1,32 +1,6 @@
-import {
- flattenHtml,
- fetchBoardJobs,
- searchGreenhouseJobs,
-} from "@/lib/scraper/greenhouse";
+import { fetchBoardJobs, searchGreenhouseJobs } from "@/lib/scraper/greenhouse";
import fixture from "./fixtures/greenhouse-anthropic.json";
-describe("flattenHtml", () => {
- it("decodes entity-encoded HTML and strips tags", () => {
- const raw =
- "&lt;div class=&quot;intro&quot;&gt;&lt;p&gt;Hello &amp; welcome&lt;/p&gt;&lt;/div&gt;";
- expect(flattenHtml(raw)).toBe("Hello & welcome");
- });
-
- it("decodes numeric entities and smart quotes", () => {
- expect(flattenHtml("Anthropic&#39;s mission")).toBe("Anthropic's mission");
- });
-
- it("collapses whitespace", () => {
- expect(flattenHtml("&lt;p&gt;a&lt;/p&gt;\n\n&lt;p&gt;b&lt;/p&gt;")).toBe(
- "a b",
- );
- });
-
- it("returns empty string for empty input", () => {
- expect(flattenHtml("")).toBe("");
- });
-});
-
describe("fetchBoardJobs", () => {
afterEach(() => {
vi.restoreAllMocks();
diff --git a/__tests__/scraper-html.spec.ts b/__tests__/scraper-html.spec.ts
new file mode 100644
index 0000000..64d7b92
--- /dev/null
+++ b/__tests__/scraper-html.spec.ts
@@ -0,0 +1,23 @@
+import { flattenHtml } from "@/lib/scraper/html";
+
+describe("flattenHtml", () => {
+ it("decodes entity-encoded HTML and strips tags", () => {
+ const raw =
+ "&lt;div class=&quot;intro&quot;&gt;&lt;p&gt;Hello &amp; welcome&lt;/p&gt;&lt;/div&gt;";
+ expect(flattenHtml(raw)).toBe("Hello & welcome");
+ });
+
+ it("decodes numeric entities and smart quotes", () => {
+ expect(flattenHtml("Anthropic&#39;s mission")).toBe("Anthropic's mission");
+ });
+
+ it("collapses whitespace", () => {
+ expect(flattenHtml("&lt;p&gt;a&lt;/p&gt;\n\n&lt;p&gt;b&lt;/p&gt;")).toBe(
+ "a b",
+ );
+ });
+
+ it("returns empty string for empty input", () => {
+ expect(flattenHtml("")).toBe("");
+ });
+});
diff --git a/src/components/automations/AtsSearchStep.tsx b/src/components/automations/AtsSearchStep.tsx
index 51a9b12..8f41498 100644
--- a/src/components/automations/AtsSearchStep.tsx
+++ b/src/components/automations/AtsSearchStep.tsx
@@ -20,7 +20,7 @@ export function AtsSearchStep({ provider, value, onChange }: AtsSearchStepProps)
const addCompany = (company: LeverCompany) => {
if (companies.some((c) => c.token === company.token)) return;
- if (companies.length >= APP_CONSTANTS.MAX_GREENHOUSE_COMPANIES) return;
+ if (companies.length >= APP_CONSTANTS.ATS_MAX_COMPANIES) return;
onChange({ ...value, companies: [...companies, company] });
};
diff --git a/src/components/automations/AutomationList.tsx b/src/components/automations/AutomationList.tsx
index 5410acf..3eb155f 100644
--- a/src/components/automations/AutomationList.tsx
+++ b/src/components/automations/AutomationList.tsx
@@ -133,13 +133,13 @@ export function AutomationList({
return null;
}
})();
- const ghCompanies: { name: string; token: string }[] =
+ const atsCompanies: { name: string; token: string }[] =
ats?.companies ?? [];
- const ghTitles: string[] = ats?.targetTitles ?? [];
- const ghLocations: string[] = ats?.locations ?? [];
- const ghKeywords: string[] = ats?.keywords ?? [];
- const ghTopK: number = ats?.topK ?? APP_CONSTANTS.MAX_JOBS_PER_RUN;
- const ghSaveUnanalyzed: boolean = ats?.saveUnanalyzed !== false;
+ const atsTitles: string[] = ats?.targetTitles ?? [];
+ const atsLocations: string[] = ats?.locations ?? [];
+ const atsKeywords: string[] = ats?.keywords ?? [];
+ const atsTopK: number = ats?.topK ?? APP_CONSTANTS.MAX_JOBS_PER_RUN;
+ const atsSaveUnanalyzed: boolean = ats?.saveUnanalyzed !== false;
return (
<div
@@ -190,14 +190,14 @@ export function AutomationList({
{!retired && (
<div className="space-y-1.5 text-sm text-muted-foreground">
<div className="flex flex-wrap items-center gap-1">
- {ghCompanies.slice(0, 3).map((c) => (
+ {atsCompanies.slice(0, 3).map((c) => (
<Badge key={c.token} variant="secondary">
{c.name}
</Badge>
))}
- {ghCompanies.length > 3 && (
+ {atsCompanies.length > 3 && (
<Badge variant="secondary">
- +{ghCompanies.length - 3} more
+ +{atsCompanies.length - 3} more
</Badge>
)}
</div>
@@ -206,33 +206,33 @@ export function AutomationList({
<span className="font-medium text-foreground">
Titles:
</span>{" "}
- {ghTitles.length ? ghTitles.join(", ") : "Not set"}
+ {atsTitles.length ? atsTitles.join(", ") : "Not set"}
</span>
<span>
<span className="font-medium text-foreground">
Keywords:
</span>{" "}
- {ghKeywords.length ? ghKeywords.join(", ") : "Any"}
+ {atsKeywords.length ? atsKeywords.join(", ") : "Any"}
</span>
<span>
<span className="font-medium text-foreground">
Location:
</span>{" "}
- {ghLocations.length
- ? ghLocations.join(", ")
+ {atsLocations.length
+ ? atsLocations.join(", ")
: "Any location"}
</span>
<span>
<span className="font-medium text-foreground">
Analyzed/run:
</span>{" "}
- {ghTopK}
+ {atsTopK}
</span>
<span>
<span className="font-medium text-foreground">
Extra listings:
</span>{" "}
- {ghSaveUnanalyzed ? "on" : "off"}
+ {atsSaveUnanalyzed ? "on" : "off"}
</span>
{automation.resume && (
<span>
diff --git a/src/components/automations/PrerankBreakdown.tsx b/src/components/automations/PrerankBreakdown.tsx
index 2265c79..20c12a9 100644
--- a/src/components/automations/PrerankBreakdown.tsx
+++ b/src/components/automations/PrerankBreakdown.tsx
@@ -19,13 +19,13 @@ export function PrerankBreakdown({
? [
{
label: "Title",
- weight: APP_CONSTANTS.GREENHOUSE_TITLE_WEIGHT,
+ weight: APP_CONSTANTS.ATS_TITLE_WEIGHT,
score: components.titleScore,
hits: components.titleHits,
},
{
label: "Keywords",
- weight: APP_CONSTANTS.GREENHOUSE_SKILL_WEIGHT,
+ weight: APP_CONSTANTS.ATS_SKILL_WEIGHT,
score: components.keywordScore,
hits: components.keywordHits,
},
diff --git a/src/components/automations/ats-search-step/CompanyPicker.tsx b/src/components/automations/ats-search-step/CompanyPicker.tsx
index 5b02a80..833098d 100644
--- a/src/components/automations/ats-search-step/CompanyPicker.tsx
+++ b/src/components/automations/ats-search-step/CompanyPicker.tsx
@@ -50,7 +50,7 @@ export function CompanyPicker({
handleListScroll,
} = useAtsCompanySearch(provider);
- const atLimit = companies.length >= APP_CONSTANTS.MAX_GREENHOUSE_COMPANIES;
+ const atLimit = companies.length >= APP_CONSTANTS.ATS_MAX_COMPANIES;
const toggleCompany = (company: LeverCompany) => {
if (companies.some((c) => c.token === company.token)) {
@@ -66,7 +66,7 @@ export function CompanyPicker({
<span>
{meta.label} Companies{" "}
<span className="font-normal text-muted-foreground">
- ({companies.length}/{APP_CONSTANTS.MAX_GREENHOUSE_COMPANIES})
+ ({companies.length}/{APP_CONSTANTS.ATS_MAX_COMPANIES})
</span>
</span>
{totalCount !== null && totalCount > 0 && (
@@ -88,7 +88,7 @@ export function CompanyPicker({
disabled={atLimit}
>
{atLimit
- ? `Max ${APP_CONSTANTS.MAX_GREENHOUSE_COMPANIES} companies reached`
+ ? `Max ${APP_CONSTANTS.ATS_MAX_COMPANIES} companies reached`
: `Search companies (e.g., ${meta.searchExample})`}
{isSearching ? (
<Loader2 className="h-4 w-4 animate-spin" />
diff --git a/src/components/automations/ats-search-step/RunOptionsFields.tsx b/src/components/automations/ats-search-step/RunOptionsFields.tsx
index 114020f..539c516 100644
--- a/src/components/automations/ats-search-step/RunOptionsFields.tsx
+++ b/src/components/automations/ats-search-step/RunOptionsFields.tsx
@@ -20,7 +20,7 @@ export function RunOptionsFields({ value, onChange }: RunOptionsFieldsProps) {
</Label>
<Slider
min={1}
- max={APP_CONSTANTS.GREENHOUSE_LISTING_CAP}
+ max={APP_CONSTANTS.ATS_LISTING_CAP}
step={1}
value={[value.topK ?? APP_CONSTANTS.MAX_JOBS_PER_RUN]}
onValueChange={(next) => onChange({ ...value, topK: next[0] })}
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index 59af825..731110f 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -98,15 +98,17 @@ export const APP_CONSTANTS = {
// Ollama runs sequentially since it serializes on the GPU).
AUTOMATION_MATCH_CONCURRENCY: 3,
+ // Shared ATS tuning (applies to every job board provider)
+ ATS_MAX_COMPANIES: 25, // per automation
+ ATS_LISTING_CAP: 50, // safety ceiling applied after the relevance floor
+ ATS_FLOOR_MIN_TITLE_HITS: 1,
+ ATS_FLOOR_MIN_KEYWORD_HITS: 1,
+ ATS_TITLE_WEIGHT: 0.6,
+ ATS_SKILL_WEIGHT: 0.4,
+
// Greenhouse job source
GREENHOUSE_BASE_URL: "https://boards-api.greenhouse.io/v1/boards",
GREENHOUSE_BOARD_URL: "https://boards.greenhouse.io", // public job board (not API)
- MAX_GREENHOUSE_COMPANIES: 25, // per automation
- GREENHOUSE_LISTING_CAP: 50, // safety ceiling applied after the relevance floor
- GREENHOUSE_FLOOR_MIN_TITLE_HITS: 1,
- GREENHOUSE_FLOOR_MIN_KEYWORD_HITS: 1,
- GREENHOUSE_TITLE_WEIGHT: 0.6,
- GREENHOUSE_SKILL_WEIGHT: 0.4,
GREENHOUSE_FETCH_TIMEOUT_MS: 25_000, // per-board AbortController timeout
GREENHOUSE_FETCH_CONCURRENCY: 5,
diff --git a/src/lib/scraper/greenhouse/pipeline.ts b/src/lib/scraper/ats/pipeline.ts
similarity index 95%
rename from src/lib/scraper/greenhouse/pipeline.ts
rename to src/lib/scraper/ats/pipeline.ts
index 72e750c..bae32eb 100644
--- a/src/lib/scraper/greenhouse/pipeline.ts
+++ b/src/lib/scraper/ats/pipeline.ts
@@ -31,14 +31,14 @@ export interface PipelineResult {
// Pure funnel: optional strict-location gate -> score -> relevance floor ->
// cap ceiling -> top-K split. No I/O, no LLM; unit-testable in isolation.
-export function runGreenhousePipeline(
+export function runAtsPipeline(
fetchedJobs: JobDetails[],
config: PipelineConfig,
resumeSkills: string[],
options?: { k?: number; cap?: number; corpus?: JobDetails[] },
): PipelineResult {
const k = options?.k ?? APP_CONSTANTS.MAX_JOBS_PER_RUN;
- const cap = options?.cap ?? APP_CONSTANTS.GREENHOUSE_LISTING_CAP;
+ const cap = options?.cap ?? APP_CONSTANTS.ATS_LISTING_CAP;
const deduped = fetchedJobs.length;
diff --git a/src/lib/scraper/greenhouse/rank.ts b/src/lib/scraper/ats/rank.ts
similarity index 95%
rename from src/lib/scraper/greenhouse/rank.ts
rename to src/lib/scraper/ats/rank.ts
index 2e45daa..ed9445f 100644
--- a/src/lib/scraper/greenhouse/rank.ts
+++ b/src/lib/scraper/ats/rank.ts
@@ -126,8 +126,8 @@ export function scoreJob(
: 0;
const score =
- APP_CONSTANTS.GREENHOUSE_TITLE_WEIGHT * titleScore +
- APP_CONSTANTS.GREENHOUSE_SKILL_WEIGHT * keywordScore +
+ APP_CONSTANTS.ATS_TITLE_WEIGHT * titleScore +
+ APP_CONSTANTS.ATS_SKILL_WEIGHT * keywordScore +
recencyTiebreak(job.postedDate);
return {
@@ -150,7 +150,7 @@ export function scoreJob(
// invariant to weight tuning.
export function passesFloor(c: PrerankComponents): boolean {
return (
- c.titleHits.length >= APP_CONSTANTS.GREENHOUSE_FLOOR_MIN_TITLE_HITS ||
- c.keywordHits.length >= APP_CONSTANTS.GREENHOUSE_FLOOR_MIN_KEYWORD_HITS
+ c.titleHits.length >= APP_CONSTANTS.ATS_FLOOR_MIN_TITLE_HITS ||
+ c.keywordHits.length >= APP_CONSTANTS.ATS_FLOOR_MIN_KEYWORD_HITS
);
}
diff --git a/src/lib/scraper/automation-run/atsRun.ts b/src/lib/scraper/automation-run/atsRun.ts
index 0ac8f5f..fec0541 100644
--- a/src/lib/scraper/automation-run/atsRun.ts
+++ b/src/lib/scraper/automation-run/atsRun.ts
@@ -3,8 +3,8 @@ import type {
FunnelStage,
} from "@/models/automation.model";
import type { AtsProvider } from "../ats/types";
-import { runGreenhousePipeline } from "../greenhouse/pipeline";
-import type { ScoredJob } from "../greenhouse/pipeline";
+import { runAtsPipeline } from "../ats/pipeline";
+import type { ScoredJob } from "../ats/pipeline";
import { dedupeJobs } from "../utils";
import { getExistingJobDedupeMap } from "@/lib/jobs/jobDedupe";
import { automationLogger } from "@/lib/automation-logger";
@@ -118,7 +118,7 @@ export async function runAtsRun(
`${label} Ranking against ${termCount} search term(s) (${config.keywords.length} keyword(s) + ${resumeSkills.length} resume skill(s))`,
);
- const pipeline = runGreenhousePipeline(dedupedJobs, config, resumeSkills, {
+ const pipeline = runAtsPipeline(dedupedJobs, config, resumeSkills, {
corpus: jobs,
k: config.topK,
});
@@ -171,7 +171,7 @@ export async function runAtsRun(
// Re-rank the pre-dedup corpus so an exhausted board (everything that
// matches is already saved) reads differently from a search that matches
// nothing at all. Only pays for itself on this zero path.
- const beforeDedup = runGreenhousePipeline(jobs, config, resumeSkills, {
+ const beforeDedup = runAtsPipeline(jobs, config, resumeSkills, {
corpus: jobs,
k: config.topK,
});
diff --git a/src/lib/scraper/automation-run/persist.ts b/src/lib/scraper/automation-run/persist.ts
index 502b736..875d3c0 100644
--- a/src/lib/scraper/automation-run/persist.ts
+++ b/src/lib/scraper/automation-run/persist.ts
@@ -9,8 +9,8 @@ import { normalizeJobUrl } from "../utils";
// matchScore column and stays below a perfect LLM score (100). Internal sort
// only — never shown as a percentage.
const PRERANK_MAX =
- APP_CONSTANTS.GREENHOUSE_TITLE_WEIGHT +
- APP_CONSTANTS.GREENHOUSE_SKILL_WEIGHT +
+ APP_CONSTANTS.ATS_TITLE_WEIGHT +
+ APP_CONSTANTS.ATS_SKILL_WEIGHT +
0.01;
export function scalePrerank(raw: number): number {
diff --git a/src/lib/scraper/greenhouse/index.ts b/src/lib/scraper/greenhouse/index.ts
index 490c6d8..98965c4 100644
--- a/src/lib/scraper/greenhouse/index.ts
+++ b/src/lib/scraper/greenhouse/index.ts
@@ -2,6 +2,7 @@ import pLimit from "p-limit";
import { APP_CONSTANTS } from "@/lib/constants";
import type { JobDetails, ScraperResult } from "../types";
import { errorReason } from "../utils";
+import { decodeHtml } from "../html";
interface GreenhouseJob {
title?: string;
@@ -18,58 +19,6 @@ interface GreenhouseBoardResponse {
meta?: { total?: number };
}
-const NAMED_ENTITIES: Record<string, string> = {
- "&lt;": "<",
- "&gt;": ">",
- "&quot;": '"',
- "&apos;": "'",
- "&#39;": "'",
- "&#x27;": "'",
- "&nbsp;": " ",
- "&rsquo;": "’",
- "&lsquo;": "‘",
- "&rdquo;": "”",
- "&ldquo;": "“",
- "&mdash;": "—",
- "&ndash;": "–",
- "&hellip;": "…",
- "&amp;": "&", // keep ampersand last so it does not undo other entities
-};
-
-function decodeEntities(input: string): string {
- let out = input;
- // Numeric entities first (decimal + hex).
- out = out.replace(/&#(\d+);/g, (_, code) =>
- String.fromCodePoint(Number(code)),
- );
- out = out.replace(/&#x([0-9a-fA-F]+);/g, (_, code) =>
- String.fromCodePoint(parseInt(code, 16)),
- );
- for (const [entity, char] of Object.entries(NAMED_ENTITIES)) {
- out = out.split(entity).join(char);
- }
- return out;
-}
-
-// Greenhouse `content` is HTML-entity-encoded HTML. Decode the entities back to
-// real tags, strip the tags, then decode once more for any entities that lived
-// inside the text, and collapse whitespace. Used for AI text processing.
-export function flattenHtml(raw: string): string {
- if (!raw) return "";
- let text = decodeEntities(raw);
- text = text.replace(/<[^>]*>/g, " ");
- text = decodeEntities(text);
- return text.replace(/\s+/g, " ").trim();
-}
-
-// Decodes entity-encoded HTML back to real HTML tags for display rendering.
-function decodeHtml(raw: string): string {
- if (!raw) return "";
- const decoded = decodeEntities(raw);
- // Second pass decodes any entities inside tag attribute values
- return decodeEntities(decoded);
-}
-
function mapGreenhouseJob(job: GreenhouseJob): JobDetails {
return {
title: (job.title ?? "").trim(),
diff --git a/src/lib/scraper/html.ts b/src/lib/scraper/html.ts
new file mode 100644
index 0000000..bdcbd01
--- /dev/null
+++ b/src/lib/scraper/html.ts
@@ -0,0 +1,53 @@
+// HTML helpers shared by ATS providers whose feeds return entity-encoded HTML.
+
+const NAMED_ENTITIES: Record<string, string> = {
+ "&lt;": "<",
+ "&gt;": ">",
+ "&quot;": '"',
+ "&apos;": "'",
+ "&#39;": "'",
+ "&#x27;": "'",
+ "&nbsp;": " ",
+ "&rsquo;": "’",
+ "&lsquo;": "‘",
+ "&rdquo;": "”",
+ "&ldquo;": "“",
+ "&mdash;": "—",
+ "&ndash;": "–",
+ "&hellip;": "…",
+ "&amp;": "&", // keep ampersand last so it does not undo other entities
+};
+
+function decodeEntities(input: string): string {
+ let out = input;
+ // Numeric entities first (decimal + hex).
+ out = out.replace(/&#(\d+);/g, (_, code) =>
+ String.fromCodePoint(Number(code)),
+ );
+ out = out.replace(/&#x([0-9a-fA-F]+);/g, (_, code) =>
+ String.fromCodePoint(parseInt(code, 16)),
+ );
+ for (const [entity, char] of Object.entries(NAMED_ENTITIES)) {
+ out = out.split(entity).join(char);
+ }
+ return out;
+}
+
+// Decode the entities back to real tags, strip the tags, then decode once more
+// for any entities that lived inside the text, and collapse whitespace. Used
+// for AI text processing.
+export function flattenHtml(raw: string): string {
+ if (!raw) return "";
+ let text = decodeEntities(raw);
+ text = text.replace(/<[^>]*>/g, " ");
+ text = decodeEntities(text);
+ return text.replace(/\s+/g, " ").trim();
+}
+
+// Decodes entity-encoded HTML back to real HTML tags for display rendering.
+export function decodeHtml(raw: string): string {
+ if (!raw) return "";
+ const decoded = decodeEntities(raw);
+ // Second pass decodes any entities inside tag attribute values
+ return decodeEntities(decoded);
+}
diff --git a/src/lib/scraper/index.ts b/src/lib/scraper/index.ts
index ea651dd..a2d0000 100644
--- a/src/lib/scraper/index.ts
+++ b/src/lib/scraper/index.ts
@@ -2,23 +2,20 @@ export * from "./types";
export * from "./utils";
export * from "./mapper";
export * from "./schedule";
-export {
- fetchBoardJobs,
- searchGreenhouseJobs,
- flattenHtml,
-} from "./greenhouse";
+export { fetchBoardJobs, searchGreenhouseJobs } from "./greenhouse";
+export { flattenHtml, decodeHtml } from "./html";
export { fetchLeverBoardJobs, searchLeverJobs } from "./lever";
export { mapLeverJob } from "./lever/mapper";
export type { LeverPosting, LeverCompany, LeverHost } from "./lever/types";
export { ATS_PROVIDERS } from "./ats/registry";
export type { AtsProvider, AtsHost } from "./ats/types";
-export { scoreJob, passesFloor, locationMatches } from "./greenhouse/rank";
+export { scoreJob, passesFloor, locationMatches } from "./ats/rank";
export {
- runGreenhousePipeline,
+ runAtsPipeline,
type PipelineConfig,
type ScoredJob,
type PipelineResult,
-} from "./greenhouse/pipeline";
+} from "./ats/pipeline";
export {
runAutomation,
getUserAiSettings,
diff --git a/src/lib/scraper/lever/mapper.ts b/src/lib/scraper/lever/mapper.ts
index db2fd0b..8360f6c 100644
--- a/src/lib/scraper/lever/mapper.ts
+++ b/src/lib/scraper/lever/mapper.ts
@@ -1,6 +1,6 @@
import type { JobDetails } from "../types";
import type { LeverPosting } from "./types";
-import { flattenHtml } from "../greenhouse";
+import { flattenHtml } from "../html";
// Lever's workplaceType string → WORKPLACE_TYPES enum key. Undefined for
// absent/unrecognized values (falls back to isRemote downstream in mapper.ts).
diff --git a/src/models/automation.schema.ts b/src/models/automation.schema.ts
index 5c7fb8e..627f1a1 100644
--- a/src/models/automation.schema.ts
+++ b/src/models/automation.schema.ts
@@ -27,12 +27,12 @@ export const GreenhouseCompanySchema = z.object({
export const GreenhouseSourceConfigSchema = z.object({
companies: z
.array(GreenhouseCompanySchema)
- .max(APP_CONSTANTS.MAX_GREENHOUSE_COMPANIES),
+ .max(APP_CONSTANTS.ATS_MAX_COMPANIES),
targetTitles: z.array(z.string().min(1).max(100)).optional(),
keywords: z.array(z.string().min(1).max(100)).optional(),
locations: z.array(z.string().min(1).max(100)).optional(),
strictLocation: z.boolean().optional(),
- topK: z.number().int().min(1).max(APP_CONSTANTS.GREENHOUSE_LISTING_CAP).optional(),
+ topK: z.number().int().min(1).max(APP_CONSTANTS.ATS_LISTING_CAP).optional(),
saveUnanalyzed: z.boolean().optional(),
});
@@ -47,7 +47,7 @@ export const LeverCompanySchema = GreenhouseCompanySchema.extend({
export const LeverSourceConfigSchema = GreenhouseSourceConfigSchema.extend({
companies: z
.array(LeverCompanySchema)
- .max(APP_CONSTANTS.MAX_GREENHOUSE_COMPANIES),
+ .max(APP_CONSTANTS.ATS_MAX_COMPANIES),
});
export const SourceConfigSchema = z.object({