Skip to content

Job application tracker and AI-powered job search assistant. Helps job seekers manage their search journey with AI resume review, job matching, task logging, and application analytics, all while keeping your data private.

Refactor pre processing step

Khuram Niaz821380356801
diff --git a/src/app/api/ai/resume/match/route.ts b/src/app/api/ai/resume/match/route.ts
index 9fd3d2c..950a911 100644
--- a/src/app/api/ai/resume/match/route.ts
+++ b/src/app/api/ai/resume/match/route.ts
@@ -12,14 +12,17 @@ import {
performSemanticSkillMatch,
calculateSemanticSimilarity,
AIUnavailableError,
+ preprocessResume,
+ preprocessJob,
} from "@/lib/ai";
-import { convertResumeToText, convertJobToText } from "@/utils/ai.utils";
import { getResumeById } from "@/actions/profile.actions";
import { getJobDetails } from "@/actions/job.actions";
import { AiModel } from "@/models/ai.model";
/**
* Job Match Endpoint
+ * Matches a resume against a job description using semantic analysis and LLM
+ * Includes preprocessing of both inputs for consistency and reliability
*/
export const POST = async (req: NextRequest) => {
const session = await auth();
@@ -35,10 +38,10 @@ export const POST = async (req: NextRequest) => {
return NextResponse.json(
{
error: `Rate limit exceeded. Try again in ${Math.ceil(
- rateLimit.resetIn / 1000
+ rateLimit.resetIn / 1000,
)} seconds.`,
},
- { status: 429 }
+ { status: 429 },
);
}
@@ -51,7 +54,7 @@ export const POST = async (req: NextRequest) => {
if (!resumeId || !jobId || !selectedModel) {
return NextResponse.json(
{ error: "Resume ID, Job ID, and model selection required" },
- { status: 400 }
+ { status: 400 },
);
}
@@ -61,8 +64,36 @@ export const POST = async (req: NextRequest) => {
getJobDetails(jobId),
]);
- const resumeText = await convertResumeToText(resume);
- const jobText = await convertJobToText(job);
+ // Preprocess both resume and job description for consistency
+ const [resumePreprocessResult, jobPreprocessResult] = await Promise.all([
+ preprocessResume(resume),
+ preprocessJob(job),
+ ]);
+
+ // Check resume preprocessing
+ if (!resumePreprocessResult.success) {
+ return NextResponse.json(
+ {
+ error: resumePreprocessResult.error.message,
+ code: resumePreprocessResult.error.code,
+ },
+ { status: 400 },
+ );
+ }
+
+ // Check job preprocessing
+ if (!jobPreprocessResult.success) {
+ return NextResponse.json(
+ {
+ error: jobPreprocessResult.error.message,
+ code: jobPreprocessResult.error.code,
+ },
+ { status: 400 },
+ );
+ }
+
+ const { normalizedText: resumeText } = resumePreprocessResult.data;
+ const { normalizedText: jobText } = jobPreprocessResult.data;
// Use semantic analysis for skill matching
const [skillMatch, similarity] = await Promise.all([
@@ -70,13 +101,13 @@ export const POST = async (req: NextRequest) => {
resumeText,
jobText,
selectedModel.provider,
- selectedModel.model || "llama3.2"
+ selectedModel.model || "llama3.2",
),
calculateSemanticSimilarity(
resumeText,
jobText,
selectedModel.provider,
- selectedModel.model || "llama3.2"
+ selectedModel.model || "llama3.2",
),
]);
@@ -98,7 +129,7 @@ Use these results in your scoring decisions.`;
const model = getModel(
selectedModel.provider,
- selectedModel.model || "llama3.2"
+ selectedModel.model || "llama3.2",
);
const result = streamText({
@@ -127,7 +158,7 @@ Use these results in your scoring decisions.`;
{
error: `Cannot connect to ${selectedModel.provider} service. Please ensure the service is running.`,
},
- { status: 503 }
+ { status: 503 },
);
}
diff --git a/src/lib/ai/index.ts b/src/lib/ai/index.ts
index 52e37b0..53284ad 100644
--- a/src/lib/ai/index.ts
+++ b/src/lib/ai/index.ts
@@ -18,14 +18,34 @@ export {
// Analysis tools
export { AIUnavailableError } from "./tools";
-// Preprocessing
+// Resume preprocessing
export {
preprocessResume,
convertResumeToText,
type PreprocessingResult,
type ResumeMetadata,
type PreprocessedResume,
-} from "./tools";
+} from "./tools/preprocessing";
+
+// Job preprocessing
+export {
+ preprocessJob,
+ convertJobToText,
+ type JobPreprocessingResult,
+ type JobMetadata,
+ type PreprocessedJob,
+} from "./tools/preprocessing-job";
+
+// Shared text processing utilities
+export {
+ removeHtmlTags,
+ normalizeWhitespace,
+ normalizeBullets,
+ normalizeHeadings,
+ extractMetadata,
+ validateText,
+ type TextMetadata,
+} from "./tools/text-processing";
// Semantic extraction for job matching (still used)
export {
diff --git a/src/lib/ai/tools/index.ts b/src/lib/ai/tools/index.ts
index e3ba86c..da4a30d 100644
--- a/src/lib/ai/tools/index.ts
+++ b/src/lib/ai/tools/index.ts
@@ -1,21 +1,50 @@
/**
- * AI Analysis Tools
- * Provide accurate counting and extraction functions for resume analysis
+ * AI Analysis Tools - Barrel Export
*
* Modules:
* - errors: Custom error classes for AI operations
- * - achievement: Extract and analyze quantified achievements
- * - formatting: Analyze text structure and formatting
- * - semantic-extraction: LLM-powered keyword and skill extraction
+ * - text-processing: Shared text normalization and metadata extraction
+ * - preprocessing: Resume preprocessing (normalization, validation)
+ * - preprocessing-job: Job description preprocessing
+ * - semantic-extraction: LLM-powered skill matching and extraction
* - semantic-similarity: LLM-powered similarity analysis
*/
// Error classes
export { AIUnavailableError } from "./errors";
-// ...existing code...
+// Shared text processing utilities
+export {
+ removeHtmlTags,
+ normalizeWhitespace,
+ normalizeBullets,
+ normalizeHeadings,
+ extractMetadata,
+ validateText,
+ type TextMetadata,
+ type ValidationError,
+ type ValidationResult,
+} from "./text-processing";
+
+// Resume preprocessing
+export {
+ preprocessResume,
+ convertResumeToText,
+ validateResume,
+ type PreprocessingResult,
+ type ResumeMetadata,
+ type PreprocessedResume,
+} from "./preprocessing";
-// ...existing code...
+// Job preprocessing
+export {
+ preprocessJob,
+ convertJobToText,
+ validateJob,
+ type JobPreprocessingResult,
+ type JobMetadata,
+ type PreprocessedJob,
+} from "./preprocessing-job";
// Semantic extraction (LLM-powered)
export { performSemanticSkillMatch } from "./semantic-extraction";
@@ -25,12 +54,3 @@ export {
calculateSemanticSimilarity,
generateMatchExplanation,
} from "./semantic-similarity";
-
-// Preprocessing
-export {
- preprocessResume,
- convertResumeToText,
- type PreprocessingResult,
- type ResumeMetadata,
- type PreprocessedResume,
-} from "./preprocessing";
diff --git a/src/lib/ai/tools/preprocessing-job.ts b/src/lib/ai/tools/preprocessing-job.ts
new file mode 100644
index 0000000..43680a0
--- /dev/null
+++ b/src/lib/ai/tools/preprocessing-job.ts
@@ -0,0 +1,157 @@
+/**
+ * Job Description Preprocessing Module
+ * Mirrors the resume preprocessing pattern for consistency
+ * Normalizes, validates, and extracts metadata from job descriptions
+ */
+
+import { JobResponse } from "@/models/job.model";
+import {
+ removeHtmlTags,
+ normalizeWhitespace,
+ normalizeBullets,
+ normalizeHeadings,
+ extractMetadata,
+ validateText,
+ type TextMetadata,
+} from "./text-processing";
+
+// TYPES
+
+export type JobMetadata = TextMetadata;
+
+export interface PreprocessedJob {
+ normalizedText: string;
+ metadata: JobMetadata;
+ isValid: boolean;
+}
+
+export type JobPreprocessingResult =
+ | { success: true; data: PreprocessedJob }
+ | {
+ success: false;
+ error: { code: string; message: string; details?: object };
+ };
+
+// VALIDATION THRESHOLDS
+
+const MIN_CHAR_COUNT = 200;
+const MAX_CHAR_COUNT = 50000;
+
+// JOB DESCRIPTION TO TEXT CONVERSION
+
+export const convertJobToText = (job: JobResponse): Promise<string> => {
+ return new Promise((resolve) => {
+ const {
+ description,
+ JobTitle: { label: jobTitle },
+ Company: { label: companyName },
+ Location: { label: location },
+ } = job;
+
+ const jobText = `
+Job Title: ${jobTitle}
+Company: ${companyName}
+Location: ${location}
+Description: ${removeHtmlTags(description)}
+ `;
+
+ return resolve(jobText);
+ });
+};
+
+// VALIDATION - Job-specific validation logic
+
+export const validateJob = (
+ text: string,
+ metadata: JobMetadata,
+): {
+ isValid: boolean;
+ error?: { code: string; message: string; details?: object };
+} => {
+ // Use shared generic validation
+ const genericValidation = validateText(
+ text,
+ MIN_CHAR_COUNT,
+ MAX_CHAR_COUNT,
+ "Job description",
+ );
+ if (!genericValidation.isValid) {
+ return genericValidation;
+ }
+
+ // Job-specific checks can be added here if needed
+ // For now, the shared validation is sufficient
+
+ return { isValid: true };
+};
+
+// MAIN ORCHESTRATOR
+
+/**
+ * Preprocess a job description
+ * Normalizes HTML, formatting, and whitespace
+ * Extracts metadata and validates content
+ *
+ * @param job - Job response object to preprocess
+ * @returns PreprocessingResult with normalized text and metadata, or error details
+ */
+export const preprocessJob = async (
+ job: JobResponse,
+): Promise<JobPreprocessingResult> => {
+ try {
+ // Convert job object to raw text
+ const rawText = await convertJobToText(job);
+
+ // Quick validation - fail fast if obviously invalid
+ if (!rawText || rawText.trim().length < MIN_CHAR_COUNT) {
+ const charCount = rawText?.trim().length || 0;
+ return {
+ success: false,
+ error: {
+ code: charCount === 0 ? "NO_CONTENT" : "TOO_SHORT",
+ message:
+ charCount === 0
+ ? "Job description appears to be empty"
+ : `Job description is too short (${charCount} characters, minimum ${MIN_CHAR_COUNT} required)`,
+ details: { characterCount: charCount },
+ },
+ };
+ }
+
+ // Apply normalization pipeline
+ let normalizedText = rawText;
+ normalizedText = normalizeWhitespace(normalizedText);
+ normalizedText = normalizeBullets(normalizedText);
+ normalizedText = normalizeHeadings(normalizedText);
+
+ // Extract metadata
+ const metadata = extractMetadata(normalizedText);
+
+ // Full validation
+ const validationResult = validateJob(normalizedText, metadata);
+ if (!validationResult.isValid) {
+ return {
+ success: false,
+ error: validationResult.error!,
+ };
+ }
+
+ return {
+ success: true,
+ data: {
+ normalizedText,
+ metadata,
+ isValid: true,
+ },
+ };
+ } catch (error) {
+ const message = error instanceof Error ? error.message : "Unknown error";
+ return {
+ success: false,
+ error: {
+ code: "PREPROCESSING_ERROR",
+ message: `Failed to preprocess job description: ${message}`,
+ },
+ };
+ }
+};
diff --git a/src/lib/ai/tools/preprocessing.ts b/src/lib/ai/tools/preprocessing.ts
index 076cae7..a2106f6 100644
--- a/src/lib/ai/tools/preprocessing.ts
+++ b/src/lib/ai/tools/preprocessing.ts
@@ -1,6 +1,7 @@
/**
* Resume Preprocessing Module
* Normalizes text, detects sections, extracts metadata, and validates resume content
+ * Uses shared text processing utilities from text-processing.ts
*/
import {
@@ -11,15 +12,19 @@ import {
SectionType,
WorkExperience,
} from "@/models/profile.model";
+import {
+ removeHtmlTags,
+ normalizeWhitespace,
+ normalizeBullets,
+ normalizeHeadings,
+ extractMetadata,
+ validateText,
+ type TextMetadata,
+} from "./text-processing";
// TYPES
-export interface ResumeMetadata {
- characterCount: number;
- wordCount: number;
- lineCount: number;
- hasContactInfo: boolean;
-}
+export type ResumeMetadata = TextMetadata;
export interface PreprocessedResume {
normalizedText: string;
@@ -36,116 +41,39 @@ export type PreprocessingResult =
// VALIDATION THRESHOLDS
-const MIN_WORD_COUNT = 50;
const MIN_CHAR_COUNT = 200;
const MAX_WORD_COUNT = 10000;
-const MAX_CONSECUTIVE_SPECIAL_CHARS = 20;
-
-// UTILITY FUNCTIONS
-
-export const removeHtmlTags = (description: string | undefined): string => {
- if (!description) return "";
- return description
- .replace(/<li[^>]*>/gi, "• ")
- .replace(/<\/(li|p|div|br)[^>]*>/gi, "\n")
- .replace(/<br\s*\/?>/gi, "\n")
- .replace(/<[^>]+>/g, " ")
- .replace(/\s+/g, " ")
- .replace(/\n\s*\n/g, "\n")
- .trim();
+// Re-export shared utilities for backward compatibility
+export {
+ removeHtmlTags,
+ normalizeWhitespace,
+ normalizeBullets,
+ normalizeHeadings,
+ extractMetadata,
};
-// NORMALIZATION FUNCTIONS
-
-export const normalizeWhitespace = (text: string): string => {
- return text
- .replace(/\r\n/g, "\n")
- .replace(/\r/g, "\n")
- .replace(/[ \t]+/g, " ")
- .replace(/\n{3,}/g, "\n\n")
- .trim();
-};
-
-export const normalizeBullets = (text: string): string => {
- return text
- .replace(/[•●○◦▪▸►◆★✦✓✔→‣⁃]/g, "•")
- .replace(/^[-–—]\s/gm, "• ")
- .replace(/^\*\s/gm, "• ");
-};
-
-export const normalizeHeadings = (text: string): string => {
- return text
- .replace(/^([A-Z][A-Z\s&]+):?\s*$/gm, (_match, heading) => {
- const normalized = heading.trim().replace(/:$/, "");
- return `\n${normalized}\n`;
- })
- .replace(/\n{3,}/g, "\n\n");
-};
-
-// METADATA EXTRACTION
-
-export const extractMetadata = (text: string): ResumeMetadata => {
- const words = text.split(/\s+/).filter((w) => w.length > 0);
- const lines = text.split("\n");
-
- return {
- characterCount: text.length,
- wordCount: words.length,
- lineCount: lines.length,
- hasContactInfo: hasContactPatterns(text),
- };
-};
-
-const hasContactPatterns = (text: string): boolean => {
- const emailPattern = /[\w.-]+@[\w.-]+\.\w+/;
- const phonePattern = /\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}/;
- return emailPattern.test(text) || phonePattern.test(text);
-};
-
-// VALIDATION
-
-interface ValidationResult {
- isValid: boolean;
- error?: { code: string; message: string; details?: object };
-}
+// VALIDATION - Resume-specific validation logic
export const validateResume = (
text: string,
metadata: ResumeMetadata,
-): ValidationResult => {
- // Check for empty content
- if (!text || text.trim().length === 0) {
- return {
- isValid: false,
- error: {
- code: "NO_CONTENT",
- message: "Resume appears to be empty or contains only whitespace",
- },
- };
- }
-
- // Check minimum length
- if (
- metadata.wordCount < MIN_WORD_COUNT ||
- metadata.characterCount < MIN_CHAR_COUNT
- ) {
- return {
- isValid: false,
- error: {
- code: "TOO_SHORT",
- message: `Resume is too short. Found ${metadata.wordCount} words and ${metadata.characterCount} characters. Minimum required: ${MIN_WORD_COUNT} words or ${MIN_CHAR_COUNT} characters.`,
- details: {
- wordCount: metadata.wordCount,
- characterCount: metadata.characterCount,
- minWordCount: MIN_WORD_COUNT,
- minCharCount: MIN_CHAR_COUNT,
- },
- },
- };
+): {
+ isValid: boolean;
+ error?: { code: string; message: string; details?: object };
+} => {
+ // Use shared generic validation first
+ const genericValidation = validateText(
+ text,
+ MIN_CHAR_COUNT,
+ MAX_WORD_COUNT * 5,
+ "Resume",
+ );
+ if (!genericValidation.isValid) {
+ return genericValidation;
}
- // Check for corruption - too long
+ // Resume-specific check: max word count
if (metadata.wordCount > MAX_WORD_COUNT) {
return {
isValid: false,
@@ -160,24 +88,6 @@ export const validateResume = (
};
}
- // Check for corruption - consecutive special characters
- const specialCharPattern = new RegExp(
- `[^a-zA-Z0-9\\s]{${MAX_CONSECUTIVE_SPECIAL_CHARS + 1},}`,
- );
- if (specialCharPattern.test(text)) {
- return {
- isValid: false,
- error: {
- code: "CORRUPTED",
- message:
- "Resume appears to be corrupted. Found excessive consecutive special characters.",
- details: {
- maxConsecutiveSpecialChars: MAX_CONSECUTIVE_SPECIAL_CHARS,
- },
- },
- };
- }
-
return { isValid: true };
};
diff --git a/src/lib/ai/tools/text-processing.ts b/src/lib/ai/tools/text-processing.ts
new file mode 100644
index 0000000..65ef229
--- /dev/null
+++ b/src/lib/ai/tools/text-processing.ts
@@ -0,0 +1,160 @@
+/**
+ * Shared Text Processing Utilities
+ * Used by both resume and job preprocessing modules
+ * Extracted from preprocessing.ts to enable code reuse
+ */
+
+// HTML AND WHITESPACE NORMALIZATION
+
+export const removeHtmlTags = (description: string | undefined): string => {
+ if (!description) return "";
+
+ return description
+ .replace(/<li[^>]*>/gi, "• ")
+ .replace(/<\/(li|p|div|br)[^>]*>/gi, "\n")
+ .replace(/<br\s*\/?>/gi, "\n")
+ .replace(/<[^>]+>/g, " ")
+ .replace(/\s+/g, " ")
+ .replace(/\n\s*\n/g, "\n")
+ .trim();
+};
+
+export const normalizeWhitespace = (text: string): string => {
+ return text
+ .replace(/\r\n/g, "\n")
+ .replace(/\r/g, "\n")
+ .replace(/[ \t]+/g, " ")
+ .replace(/\n{3,}/g, "\n\n")
+ .trim();
+};
+
+export const normalizeBullets = (text: string): string => {
+ return text
+ .replace(/[•●○◦▪▸►◆★✦✓✔→‣⁃]/g, "•")
+ .replace(/^[-–—]\s/gm, "• ")
+ .replace(/^\*\s/gm, "• ");
+};
+
+export const normalizeHeadings = (text: string): string => {
+ return text
+ .replace(/^([A-Z][A-Z\s&]+):?\s*$/gm, (_match, heading) => {
+ const normalized = heading.trim().replace(/:$/, "");
+ return `\n${normalized}\n`;
+ })
+ .replace(/\n{3,}/g, "\n\n");
+};
+
+// METADATA EXTRACTION
+
+export interface TextMetadata {
+ characterCount: number;
+ wordCount: number;
+ lineCount: number;
+ hasContactInfo: boolean;
+}
+
+export const extractMetadata = (text: string): TextMetadata => {
+ const words = text.split(/\s+/).filter((w) => w.length > 0);
+ const lines = text.split("\n");
+
+ return {
+ characterCount: text.length,
+ wordCount: words.length,
+ lineCount: lines.length,
+ hasContactInfo: hasContactPatterns(text),
+ };
+};
+
+const hasContactPatterns = (text: string): boolean => {
+ const emailPattern = /[\w.-]+@[\w.-]+\.\w+/;
+ const phonePattern = /\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}/;
+ return emailPattern.test(text) || phonePattern.test(text);
+};
+
+// VALIDATION HELPERS
+
+export interface ValidationError {
+ code: string;
+ message: string;
+ details?: object;
+}
+
+export interface ValidationResult {
+ isValid: boolean;
+ error?: ValidationError;
+}
+
+/**
+ * Generic text validation - checks for empty content and basic corruption
+ * @param text - Text to validate
+ * @param minCharCount - Minimum character count required
+ * @param maxCharCount - Maximum character count allowed
+ * @param contextLabel - Label for error messages (e.g., "Resume", "Job description")
+ */
+export const validateText = (
+ text: string,
+ minCharCount: number = 200,
+ maxCharCount: number = 50000,
+ contextLabel: string = "Content",
+): ValidationResult => {
+ // Check for empty content
+ if (!text || text.trim().length === 0) {
+ return {
+ isValid: false,
+ error: {
+ code: "NO_CONTENT",
+ message: `${contextLabel} appears to be empty or contains only whitespace`,
+ },
+ };
+ }
+
+ // Check minimum length
+ if (text.length < minCharCount) {
+ return {
+ isValid: false,
+ error: {
+ code: "TOO_SHORT",
+ message: `${contextLabel} is too short. Found ${text.length} characters, minimum required: ${minCharCount} characters.`,
+ details: {
+ characterCount: text.length,
+ minCharCount,
+ },
+ },
+ };
+ }
+
+ // Check maximum length
+ if (text.length > maxCharCount) {
+ return {
+ isValid: false,
+ error: {
+ code: "TOO_LONG",
+ message: `${contextLabel} is too long. Found ${text.length} characters, maximum allowed: ${maxCharCount} characters.`,
+ details: {
+ characterCount: text.length,
+ maxCharCount,
+ },
+ },
+ };
+ }
+
+ // Check for corruption - consecutive special characters
+ const MAX_CONSECUTIVE_SPECIAL_CHARS = 20;
+ const specialCharPattern = new RegExp(
+ `[^a-zA-Z0-9\\s]{${MAX_CONSECUTIVE_SPECIAL_CHARS + 1},}`,
+ );
+ if (specialCharPattern.test(text)) {
+ return {
+ isValid: false,
+ error: {
+ code: "CORRUPTED",
+ message: `${contextLabel} appears to be corrupted. Found excessive consecutive special characters.`,
+ details: {
+ maxConsecutiveSpecialChars: MAX_CONSECUTIVE_SPECIAL_CHARS,
+ },
+ },
+ };
+ }
+
+ return { isValid: true };
+};
diff --git a/src/utils/ai.utils.ts b/src/utils/ai.utils.ts
index 89b3964..9c0b54d 100644
--- a/src/utils/ai.utils.ts
+++ b/src/utils/ai.utils.ts
@@ -3,21 +3,7 @@ import { AiProvider } from "@/models/ai.model";
// Re-export for backwards compatibility
export { convertResumeToText } from "@/lib/ai/tools/preprocessing";
-
-const removeHtmlTags = (description: string | undefined): string => {
- if (!description) return "";
-
- return (
- description
- .replace(/<li[^>]*>/gi, "• ")
- .replace(/<\/(li|p|div|br)[^>]*>/gi, "\n")
- .replace(/<br\s*\/?>/gi, "\n")
- .replace(/<[^>]+>/g, " ")
- .replace(/\s+/g, " ")
- .replace(/\n\s*\n/g, "\n")
- .trim()
- );
-};
+export { convertJobToText } from "@/lib/ai/tools/preprocessing-job";
export interface ModelCheckResult {
isRunning: boolean;
@@ -33,7 +19,7 @@ export interface ModelCheckResult {
*/
export const checkIfModelIsRunning = async (
modelName: string | undefined,
- provider: AiProvider
+ provider: AiProvider,
): Promise<ModelCheckResult> => {
// Only check for Ollama provider
if (provider !== AiProvider.OLLAMA) {
@@ -122,23 +108,3 @@ export const fetchRunningModels = async (): Promise<{
};
}
};
-
-export const convertJobToText = (job: JobResponse): Promise<string> => {
- return new Promise((resolve) => {
- const {
- description,
- JobTitle: { label: jobTitle },
- Company: { label: companyName },
- Location: { label: location },
- } = job;
-
- const jobText = `
- Job Title: ${jobTitle}
- Company: ${companyName}
- Location: ${location}
- Description: ${removeHtmlTags(description)}
- `;
-
- return resolve(jobText);
- });
-};