Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.
TypeScript ai-agentai-resume-analyzerapplication-trackerapplication-tracking-systematscareerjob-applicationjob-automationjob-huntingjob-searchjob-seeker-toolsjob-trackerjob-trackingjobsearchjobseekersmcpollamaresumeresume-reviewself-hosted
Refactor pre processing step
Khuram Niaz821380356801
diff --git a/src/app/api/ai/resume/match/route.ts b/src/app/api/ai/resume/match/route.tsindex 9fd3d2c..950a911 100644--- a/src/app/api/ai/resume/match/route.ts+++ b/src/app/api/ai/resume/match/route.ts@@ -12,14 +12,17 @@ import {performSemanticSkillMatch,calculateSemanticSimilarity,AIUnavailableError,+ preprocessResume,+ preprocessJob,} from "@/lib/ai";-import { convertResumeToText, convertJobToText } from "@/utils/ai.utils";import { getResumeById } from "@/actions/profile.actions";import { getJobDetails } from "@/actions/job.actions";import { AiModel } from "@/models/ai.model";/*** Job Match Endpoint+ * Matches a resume against a job description using semantic analysis and LLM+ * Includes preprocessing of both inputs for consistency and reliability*/export const POST = async (req: NextRequest) => {const session = await auth();@@ -35,10 +38,10 @@ export const POST = async (req: NextRequest) => {return NextResponse.json({error: `Rate limit exceeded. Try again in ${Math.ceil(- rateLimit.resetIn / 1000+ rateLimit.resetIn / 1000,)} seconds.`,},- { status: 429 }+ { status: 429 },);}@@ -51,7 +54,7 @@ export const POST = async (req: NextRequest) => {if (!resumeId || !jobId || !selectedModel) {return NextResponse.json({ error: "Resume ID, Job ID, and model selection required" },- { status: 400 }+ { status: 400 },);}@@ -61,8 +64,36 @@ export const POST = async (req: NextRequest) => {getJobDetails(jobId),]);- const resumeText = await convertResumeToText(resume);- const jobText = await convertJobToText(job);+ // Preprocess both resume and job description for consistency+ const [resumePreprocessResult, jobPreprocessResult] = await Promise.all([+ preprocessResume(resume),+ preprocessJob(job),+ ]);++ // Check resume preprocessing+ if (!resumePreprocessResult.success) {+ return NextResponse.json(+ {+ error: resumePreprocessResult.error.message,+ code: resumePreprocessResult.error.code,+ },+ { status: 400 },+ );+ }++ // Check job preprocessing+ if (!jobPreprocessResult.success) {+ return NextResponse.json(+ {+ error: jobPreprocessResult.error.message,+ code: jobPreprocessResult.error.code,+ },+ { status: 400 },+ );+ }++ const { normalizedText: resumeText } = resumePreprocessResult.data;+ const { normalizedText: jobText } = jobPreprocessResult.data;// Use semantic analysis for skill matchingconst [skillMatch, similarity] = await Promise.all([@@ -70,13 +101,13 @@ export const POST = async (req: NextRequest) => {resumeText,jobText,selectedModel.provider,- selectedModel.model || "llama3.2"+ selectedModel.model || "llama3.2",),calculateSemanticSimilarity(resumeText,jobText,selectedModel.provider,- selectedModel.model || "llama3.2"+ selectedModel.model || "llama3.2",),]);@@ -98,7 +129,7 @@ Use these results in your scoring decisions.`;const model = getModel(selectedModel.provider,- selectedModel.model || "llama3.2"+ selectedModel.model || "llama3.2",);const result = streamText({@@ -127,7 +158,7 @@ Use these results in your scoring decisions.`;{error: `Cannot connect to ${selectedModel.provider} service. Please ensure the service is running.`,},- { status: 503 }+ { status: 503 },);}diff --git a/src/lib/ai/index.ts b/src/lib/ai/index.tsindex 52e37b0..53284ad 100644--- a/src/lib/ai/index.ts+++ b/src/lib/ai/index.ts@@ -18,14 +18,34 @@ export {// Analysis toolsexport { AIUnavailableError } from "./tools";-// Preprocessing+// Resume preprocessingexport {preprocessResume,convertResumeToText,type PreprocessingResult,type ResumeMetadata,type PreprocessedResume,-} from "./tools";+} from "./tools/preprocessing";++// Job preprocessing+export {+ preprocessJob,+ convertJobToText,+ type JobPreprocessingResult,+ type JobMetadata,+ type PreprocessedJob,+} from "./tools/preprocessing-job";++// Shared text processing utilities+export {+ removeHtmlTags,+ normalizeWhitespace,+ normalizeBullets,+ normalizeHeadings,+ extractMetadata,+ validateText,+ type TextMetadata,+} from "./tools/text-processing";// Semantic extraction for job matching (still used)export {diff --git a/src/lib/ai/tools/index.ts b/src/lib/ai/tools/index.tsindex e3ba86c..da4a30d 100644--- a/src/lib/ai/tools/index.ts+++ b/src/lib/ai/tools/index.ts@@ -1,21 +1,50 @@/**- * AI Analysis Tools- * Provide accurate counting and extraction functions for resume analysis+ * AI Analysis Tools - Barrel Export** Modules:* - errors: Custom error classes for AI operations- * - achievement: Extract and analyze quantified achievements- * - formatting: Analyze text structure and formatting- * - semantic-extraction: LLM-powered keyword and skill extraction+ * - text-processing: Shared text normalization and metadata extraction+ * - preprocessing: Resume preprocessing (normalization, validation)+ * - preprocessing-job: Job description preprocessing+ * - semantic-extraction: LLM-powered skill matching and extraction* - semantic-similarity: LLM-powered similarity analysis*/// Error classesexport { AIUnavailableError } from "./errors";-// ...existing code...+// Shared text processing utilities+export {+ removeHtmlTags,+ normalizeWhitespace,+ normalizeBullets,+ normalizeHeadings,+ extractMetadata,+ validateText,+ type TextMetadata,+ type ValidationError,+ type ValidationResult,+} from "./text-processing";++// Resume preprocessing+export {+ preprocessResume,+ convertResumeToText,+ validateResume,+ type PreprocessingResult,+ type ResumeMetadata,+ type PreprocessedResume,+} from "./preprocessing";-// ...existing code...+// Job preprocessing+export {+ preprocessJob,+ convertJobToText,+ validateJob,+ type JobPreprocessingResult,+ type JobMetadata,+ type PreprocessedJob,+} from "./preprocessing-job";// Semantic extraction (LLM-powered)export { performSemanticSkillMatch } from "./semantic-extraction";@@ -25,12 +54,3 @@ export {calculateSemanticSimilarity,generateMatchExplanation,} from "./semantic-similarity";--// Preprocessing-export {- preprocessResume,- convertResumeToText,- type PreprocessingResult,- type ResumeMetadata,- type PreprocessedResume,-} from "./preprocessing";diff --git a/src/lib/ai/tools/preprocessing-job.ts b/src/lib/ai/tools/preprocessing-job.tsnew file mode 100644index 0000000..43680a0--- /dev/null+++ b/src/lib/ai/tools/preprocessing-job.ts@@ -0,0 +1,157 @@+/**+ * Job Description Preprocessing Module+ * Mirrors the resume preprocessing pattern for consistency+ * Normalizes, validates, and extracts metadata from job descriptions+ */++import { JobResponse } from "@/models/job.model";+import {+ removeHtmlTags,+ normalizeWhitespace,+ normalizeBullets,+ normalizeHeadings,+ extractMetadata,+ validateText,+ type TextMetadata,+} from "./text-processing";++// TYPES++export type JobMetadata = TextMetadata;++export interface PreprocessedJob {+ normalizedText: string;+ metadata: JobMetadata;+ isValid: boolean;+}++export type JobPreprocessingResult =+ | { success: true; data: PreprocessedJob }+ | {+ success: false;+ error: { code: string; message: string; details?: object };+ };++// VALIDATION THRESHOLDS++const MIN_CHAR_COUNT = 200;+const MAX_CHAR_COUNT = 50000;++// JOB DESCRIPTION TO TEXT CONVERSION++export const convertJobToText = (job: JobResponse): Promise<string> => {+ return new Promise((resolve) => {+ const {+ description,+ JobTitle: { label: jobTitle },+ Company: { label: companyName },+ Location: { label: location },+ } = job;++ const jobText = `+Job Title: ${jobTitle}+Company: ${companyName}+Location: ${location}+Description: ${removeHtmlTags(description)}+ `;++ return resolve(jobText);+ });+};++// VALIDATION - Job-specific validation logic++export const validateJob = (+ text: string,+ metadata: JobMetadata,+): {+ isValid: boolean;+ error?: { code: string; message: string; details?: object };+} => {+ // Use shared generic validation+ const genericValidation = validateText(+ text,+ MIN_CHAR_COUNT,+ MAX_CHAR_COUNT,+ "Job description",+ );+ if (!genericValidation.isValid) {+ return genericValidation;+ }++ // Job-specific checks can be added here if needed+ // For now, the shared validation is sufficient++ return { isValid: true };+};++// MAIN ORCHESTRATOR++/**+ * Preprocess a job description+ * Normalizes HTML, formatting, and whitespace+ * Extracts metadata and validates content+ *+ * @param job - Job response object to preprocess+ * @returns PreprocessingResult with normalized text and metadata, or error details+ */+export const preprocessJob = async (+ job: JobResponse,+): Promise<JobPreprocessingResult> => {+ try {+ // Convert job object to raw text+ const rawText = await convertJobToText(job);++ // Quick validation - fail fast if obviously invalid+ if (!rawText || rawText.trim().length < MIN_CHAR_COUNT) {+ const charCount = rawText?.trim().length || 0;+ return {+ success: false,+ error: {+ code: charCount === 0 ? "NO_CONTENT" : "TOO_SHORT",+ message:+ charCount === 0+ ? "Job description appears to be empty"+ : `Job description is too short (${charCount} characters, minimum ${MIN_CHAR_COUNT} required)`,+ details: { characterCount: charCount },+ },+ };+ }++ // Apply normalization pipeline+ let normalizedText = rawText;+ normalizedText = normalizeWhitespace(normalizedText);+ normalizedText = normalizeBullets(normalizedText);+ normalizedText = normalizeHeadings(normalizedText);++ // Extract metadata+ const metadata = extractMetadata(normalizedText);++ // Full validation+ const validationResult = validateJob(normalizedText, metadata);+ if (!validationResult.isValid) {+ return {+ success: false,+ error: validationResult.error!,+ };+ }++ return {+ success: true,+ data: {+ normalizedText,+ metadata,+ isValid: true,+ },+ };+ } catch (error) {+ const message = error instanceof Error ? error.message : "Unknown error";+ return {+ success: false,+ error: {+ code: "PREPROCESSING_ERROR",+ message: `Failed to preprocess job description: ${message}`,+ },+ };+ }+};diff --git a/src/lib/ai/tools/preprocessing.ts b/src/lib/ai/tools/preprocessing.tsindex 076cae7..a2106f6 100644--- a/src/lib/ai/tools/preprocessing.ts+++ b/src/lib/ai/tools/preprocessing.ts@@ -1,6 +1,7 @@/*** Resume Preprocessing Module* Normalizes text, detects sections, extracts metadata, and validates resume content+ * Uses shared text processing utilities from text-processing.ts*/import {@@ -11,15 +12,19 @@ import {SectionType,WorkExperience,} from "@/models/profile.model";+import {+ removeHtmlTags,+ normalizeWhitespace,+ normalizeBullets,+ normalizeHeadings,+ extractMetadata,+ validateText,+ type TextMetadata,+} from "./text-processing";// TYPES-export interface ResumeMetadata {- characterCount: number;- wordCount: number;- lineCount: number;- hasContactInfo: boolean;-}+export type ResumeMetadata = TextMetadata;export interface PreprocessedResume {normalizedText: string;@@ -36,116 +41,39 @@ export type PreprocessingResult =// VALIDATION THRESHOLDS-const MIN_WORD_COUNT = 50;const MIN_CHAR_COUNT = 200;const MAX_WORD_COUNT = 10000;-const MAX_CONSECUTIVE_SPECIAL_CHARS = 20;--// UTILITY FUNCTIONS--export const removeHtmlTags = (description: string | undefined): string => {- if (!description) return "";- return description- .replace(/<li[^>]*>/gi, "• ")- .replace(/<\/(li|p|div|br)[^>]*>/gi, "\n")- .replace(/<br\s*\/?>/gi, "\n")- .replace(/<[^>]+>/g, " ")- .replace(/\s+/g, " ")- .replace(/\n\s*\n/g, "\n")- .trim();+// Re-export shared utilities for backward compatibility+export {+ removeHtmlTags,+ normalizeWhitespace,+ normalizeBullets,+ normalizeHeadings,+ extractMetadata,};-// NORMALIZATION FUNCTIONS--export const normalizeWhitespace = (text: string): string => {- return text- .replace(/\r\n/g, "\n")- .replace(/\r/g, "\n")- .replace(/[ \t]+/g, " ")- .replace(/\n{3,}/g, "\n\n")- .trim();-};--export const normalizeBullets = (text: string): string => {- return text- .replace(/[•●○◦▪▸►◆★✦✓✔→‣⁃]/g, "•")- .replace(/^[-–—]\s/gm, "• ")- .replace(/^\*\s/gm, "• ");-};--export const normalizeHeadings = (text: string): string => {- return text- .replace(/^([A-Z][A-Z\s&]+):?\s*$/gm, (_match, heading) => {- const normalized = heading.trim().replace(/:$/, "");- return `\n${normalized}\n`;- })- .replace(/\n{3,}/g, "\n\n");-};--// METADATA EXTRACTION--export const extractMetadata = (text: string): ResumeMetadata => {- const words = text.split(/\s+/).filter((w) => w.length > 0);- const lines = text.split("\n");-- return {- characterCount: text.length,- wordCount: words.length,- lineCount: lines.length,- hasContactInfo: hasContactPatterns(text),- };-};--const hasContactPatterns = (text: string): boolean => {- const emailPattern = /[\w.-]+@[\w.-]+\.\w+/;- const phonePattern = /\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}/;- return emailPattern.test(text) || phonePattern.test(text);-};--// VALIDATION--interface ValidationResult {- isValid: boolean;- error?: { code: string; message: string; details?: object };-}+// VALIDATION - Resume-specific validation logicexport const validateResume = (text: string,metadata: ResumeMetadata,-): ValidationResult => {- // Check for empty content- if (!text || text.trim().length === 0) {- return {- isValid: false,- error: {- code: "NO_CONTENT",- message: "Resume appears to be empty or contains only whitespace",- },- };- }-- // Check minimum length- if (- metadata.wordCount < MIN_WORD_COUNT ||- metadata.characterCount < MIN_CHAR_COUNT- ) {- return {- isValid: false,- error: {- code: "TOO_SHORT",- message: `Resume is too short. Found ${metadata.wordCount} words and ${metadata.characterCount} characters. Minimum required: ${MIN_WORD_COUNT} words or ${MIN_CHAR_COUNT} characters.`,- details: {- wordCount: metadata.wordCount,- characterCount: metadata.characterCount,- minWordCount: MIN_WORD_COUNT,- minCharCount: MIN_CHAR_COUNT,- },- },- };+): {+ isValid: boolean;+ error?: { code: string; message: string; details?: object };+} => {+ // Use shared generic validation first+ const genericValidation = validateText(+ text,+ MIN_CHAR_COUNT,+ MAX_WORD_COUNT * 5,+ "Resume",+ );+ if (!genericValidation.isValid) {+ return genericValidation;}- // Check for corruption - too long+ // Resume-specific check: max word countif (metadata.wordCount > MAX_WORD_COUNT) {return {isValid: false,@@ -160,24 +88,6 @@ export const validateResume = (};}- // Check for corruption - consecutive special characters- const specialCharPattern = new RegExp(- `[^a-zA-Z0-9\\s]{${MAX_CONSECUTIVE_SPECIAL_CHARS + 1},}`,- );- if (specialCharPattern.test(text)) {- return {- isValid: false,- error: {- code: "CORRUPTED",- message:- "Resume appears to be corrupted. Found excessive consecutive special characters.",- details: {- maxConsecutiveSpecialChars: MAX_CONSECUTIVE_SPECIAL_CHARS,- },- },- };- }-return { isValid: true };};diff --git a/src/lib/ai/tools/text-processing.ts b/src/lib/ai/tools/text-processing.tsnew file mode 100644index 0000000..65ef229--- /dev/null+++ b/src/lib/ai/tools/text-processing.ts@@ -0,0 +1,160 @@+/**+ * Shared Text Processing Utilities+ * Used by both resume and job preprocessing modules+ * Extracted from preprocessing.ts to enable code reuse+ */++// HTML AND WHITESPACE NORMALIZATION++export const removeHtmlTags = (description: string | undefined): string => {+ if (!description) return "";++ return description+ .replace(/<li[^>]*>/gi, "• ")+ .replace(/<\/(li|p|div|br)[^>]*>/gi, "\n")+ .replace(/<br\s*\/?>/gi, "\n")+ .replace(/<[^>]+>/g, " ")+ .replace(/\s+/g, " ")+ .replace(/\n\s*\n/g, "\n")+ .trim();+};++export const normalizeWhitespace = (text: string): string => {+ return text+ .replace(/\r\n/g, "\n")+ .replace(/\r/g, "\n")+ .replace(/[ \t]+/g, " ")+ .replace(/\n{3,}/g, "\n\n")+ .trim();+};++export const normalizeBullets = (text: string): string => {+ return text+ .replace(/[•●○◦▪▸►◆★✦✓✔→‣⁃]/g, "•")+ .replace(/^[-–—]\s/gm, "• ")+ .replace(/^\*\s/gm, "• ");+};++export const normalizeHeadings = (text: string): string => {+ return text+ .replace(/^([A-Z][A-Z\s&]+):?\s*$/gm, (_match, heading) => {+ const normalized = heading.trim().replace(/:$/, "");+ return `\n${normalized}\n`;+ })+ .replace(/\n{3,}/g, "\n\n");+};++// METADATA EXTRACTION++export interface TextMetadata {+ characterCount: number;+ wordCount: number;+ lineCount: number;+ hasContactInfo: boolean;+}++export const extractMetadata = (text: string): TextMetadata => {+ const words = text.split(/\s+/).filter((w) => w.length > 0);+ const lines = text.split("\n");++ return {+ characterCount: text.length,+ wordCount: words.length,+ lineCount: lines.length,+ hasContactInfo: hasContactPatterns(text),+ };+};++const hasContactPatterns = (text: string): boolean => {+ const emailPattern = /[\w.-]+@[\w.-]+\.\w+/;+ const phonePattern = /\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}/;+ return emailPattern.test(text) || phonePattern.test(text);+};++// VALIDATION HELPERS++export interface ValidationError {+ code: string;+ message: string;+ details?: object;+}++export interface ValidationResult {+ isValid: boolean;+ error?: ValidationError;+}++/**+ * Generic text validation - checks for empty content and basic corruption+ * @param text - Text to validate+ * @param minCharCount - Minimum character count required+ * @param maxCharCount - Maximum character count allowed+ * @param contextLabel - Label for error messages (e.g., "Resume", "Job description")+ */+export const validateText = (+ text: string,+ minCharCount: number = 200,+ maxCharCount: number = 50000,+ contextLabel: string = "Content",+): ValidationResult => {+ // Check for empty content+ if (!text || text.trim().length === 0) {+ return {+ isValid: false,+ error: {+ code: "NO_CONTENT",+ message: `${contextLabel} appears to be empty or contains only whitespace`,+ },+ };+ }++ // Check minimum length+ if (text.length < minCharCount) {+ return {+ isValid: false,+ error: {+ code: "TOO_SHORT",+ message: `${contextLabel} is too short. Found ${text.length} characters, minimum required: ${minCharCount} characters.`,+ details: {+ characterCount: text.length,+ minCharCount,+ },+ },+ };+ }++ // Check maximum length+ if (text.length > maxCharCount) {+ return {+ isValid: false,+ error: {+ code: "TOO_LONG",+ message: `${contextLabel} is too long. Found ${text.length} characters, maximum allowed: ${maxCharCount} characters.`,+ details: {+ characterCount: text.length,+ maxCharCount,+ },+ },+ };+ }++ // Check for corruption - consecutive special characters+ const MAX_CONSECUTIVE_SPECIAL_CHARS = 20;+ const specialCharPattern = new RegExp(+ `[^a-zA-Z0-9\\s]{${MAX_CONSECUTIVE_SPECIAL_CHARS + 1},}`,+ );+ if (specialCharPattern.test(text)) {+ return {+ isValid: false,+ error: {+ code: "CORRUPTED",+ message: `${contextLabel} appears to be corrupted. Found excessive consecutive special characters.`,+ details: {+ maxConsecutiveSpecialChars: MAX_CONSECUTIVE_SPECIAL_CHARS,+ },+ },+ };+ }++ return { isValid: true };+};diff --git a/src/utils/ai.utils.ts b/src/utils/ai.utils.tsindex 89b3964..9c0b54d 100644--- a/src/utils/ai.utils.ts+++ b/src/utils/ai.utils.ts@@ -3,21 +3,7 @@ import { AiProvider } from "@/models/ai.model";// Re-export for backwards compatibilityexport { convertResumeToText } from "@/lib/ai/tools/preprocessing";--const removeHtmlTags = (description: string | undefined): string => {- if (!description) return "";-- return (- description- .replace(/<li[^>]*>/gi, "• ")- .replace(/<\/(li|p|div|br)[^>]*>/gi, "\n")- .replace(/<br\s*\/?>/gi, "\n")- .replace(/<[^>]+>/g, " ")- .replace(/\s+/g, " ")- .replace(/\n\s*\n/g, "\n")- .trim()- );-};+export { convertJobToText } from "@/lib/ai/tools/preprocessing-job";export interface ModelCheckResult {isRunning: boolean;@@ -33,7 +19,7 @@ export interface ModelCheckResult {*/export const checkIfModelIsRunning = async (modelName: string | undefined,- provider: AiProvider+ provider: AiProvider,): Promise<ModelCheckResult> => {// Only check for Ollama providerif (provider !== AiProvider.OLLAMA) {@@ -122,23 +108,3 @@ export const fetchRunningModels = async (): Promise<{};}};--export const convertJobToText = (job: JobResponse): Promise<string> => {- return new Promise((resolve) => {- const {- description,- JobTitle: { label: jobTitle },- Company: { label: companyName },- Location: { label: location },- } = job;-- const jobText = `- Job Title: ${jobTitle}- Company: ${companyName}- Location: ${location}- Description: ${removeHtmlTags(description)}- `;-- return resolve(jobText);- });-};