Skip to content

Open-source status page, incident response and uptime monitoring

chore: improve cmdk search results (#2271)

Maximilian Kaske97c728cfec23
diff --git a/apps/web/src/content/cmdk.tsx b/apps/web/src/content/cmdk.tsx
index cdc572efc..da40c8f8e 100644
--- a/apps/web/src/content/cmdk.tsx
+++ b/apps/web/src/content/cmdk.tsx
@@ -29,6 +29,7 @@ import {
type Corpus,
type SearchResult,
} from "@/content/search-meta";
+import { buildHighlightRegex } from "@/content/utils/search-match";
type ConfigItem = {
type: "item";
@@ -526,10 +527,7 @@ function SearchResults({
const router = useRouter();
const highlightRe = React.useMemo(
- () =>
- search
- ? new RegExp(search.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i")
- : null,
+ () => buildHighlightRegex(search),
[search],
);
const highlight = (text: string) =>
@@ -541,7 +539,6 @@ function SearchResults({
<CommandItem
key={item.href}
value={item.href}
- keywords={[item.metadata.title, item.content, search]}
onSelect={() => {
router.push(item.href);
setOpen(false);
diff --git a/apps/web/src/content/highlight-text.tsx b/apps/web/src/content/highlight-text.tsx
index 02324dbfc..d3a5e5a6a 100644
--- a/apps/web/src/content/highlight-text.tsx
+++ b/apps/web/src/content/highlight-text.tsx
@@ -3,10 +3,18 @@
import { useSearchParams } from "next/navigation";
import { useEffect, useRef } from "react";
+import { buildHighlightRegex } from "@/content/utils/search-match";
+
+function escapeHtml(input: string) {
+ return input.replace(/[&<>]/g, (c) =>
+ c === "&" ? "&amp;" : c === "<" ? "&lt;" : "&gt;",
+ );
+}
+
function highlight(root: HTMLElement, query: string) {
- if (!query) return;
+ const regex = buildHighlightRegex(query);
+ if (!regex) return;
- const regex = new RegExp(`(${query})`, "gi");
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, null);
const textNodes: Text[] = [];
@@ -21,10 +29,16 @@ function highlight(root: HTMLElement, query: string) {
for (const textNode of textNodes) {
const nodeValue = textNode.nodeValue;
+ // `g`-flag .test is stateful — reset so a hit in one node can't skip the next.
+ regex.lastIndex = 0;
if (!nodeValue || !regex.test(nodeValue)) continue;
const span = document.createElement("span");
- span.innerHTML = nodeValue.replace(regex, "<mark>$1</mark>");
+ // Escape first — the text node's value must not parse as HTML.
+ span.innerHTML = escapeHtml(nodeValue).replace(
+ regex,
+ (match) => `<mark>${match}</mark>`,
+ );
textNode.parentNode?.replaceChild(span, textNode);
}
diff --git a/apps/web/src/content/utils/search-index.ts b/apps/web/src/content/utils/search-index.ts
index 39f4ee687..357bc83ee 100644
--- a/apps/web/src/content/utils/search-index.ts
+++ b/apps/web/src/content/utils/search-index.ts
@@ -5,8 +5,11 @@ import { type MDXData, PAGE_TYPES, getHomePage, getPages } from ".";
import type { Corpus, SearchResult } from "../search-meta";
import {
closestHeadingSlug,
+ filterTerms,
+ findCorrection,
getContentSnippet,
makeMatcher,
+ normalizeForMatch,
sanitizeContent,
scoreDoc,
} from "./search-match";
@@ -22,6 +25,7 @@ type IndexedDoc = {
sanitizedLower: string;
headings: Heading[];
headingsLower: string;
+ descriptionLower: string;
};
const CORPORA: Corpus[] = PAGE_TYPES.filter((t): t is Corpus => t !== "all");
@@ -30,6 +34,7 @@ const CORPORA: Corpus[] = PAGE_TYPES.filter((t): t is Corpus => t !== "all");
// in dev so editing an .mdx file is reflected without a restart.
const shouldCache = process.env.NODE_ENV === "production";
const cache = new Map<Corpus, IndexedDoc[]>();
+const vocabCache = new Map<Corpus, string[]>();
function homeDoc(): MDXData {
const home = getHomePage();
@@ -40,7 +45,10 @@ function homeDoc(): MDXData {
function indexDoc(doc: MDXData, type: Corpus): IndexedDoc {
const raw = doc.content;
- const sanitized = sanitizeContent(raw);
+ const faqText = (doc.metadata.faq ?? [])
+ .map((f) => `${f.question} ${f.answer}`)
+ .join(" ");
+ const sanitized = sanitizeContent(faqText ? `${raw} ${faqText}` : raw);
const headings: Heading[] = [];
const headingTexts: string[] = [];
const headingRegex = /^#{1,6}\s+(.+)$/gm;
@@ -54,12 +62,13 @@ function indexDoc(doc: MDXData, type: Corpus): IndexedDoc {
return {
doc,
type,
- titleLower: doc.metadata.title.toLowerCase(),
- rawLower: raw.toLowerCase(),
+ titleLower: normalizeForMatch(doc.metadata.title),
+ rawLower: normalizeForMatch(raw),
sanitized,
- sanitizedLower: sanitized.toLowerCase(),
+ sanitizedLower: normalizeForMatch(sanitized),
headings,
- headingsLower: headingTexts.join(" \n ").toLowerCase(),
+ headingsLower: normalizeForMatch(headingTexts.join(" \n ")),
+ descriptionLower: normalizeForMatch(doc.metadata.description),
};
}
@@ -78,6 +87,30 @@ function getCorpus(type: Corpus): IndexedDoc[] {
return indexed;
}
+function buildVocab(entries: IndexedDoc[]): string[] {
+ const seen = new Set<string>();
+ for (const e of entries) {
+ for (const w of `${e.titleLower} ${e.headingsLower}`.split(
+ /[^\p{L}\p{N}]+/u,
+ )) {
+ if (w.length >= 4) seen.add(w);
+ }
+ }
+ return [...seen].sort();
+}
+
+function getVocab(p: Corpus | "all"): string[] {
+ if (p === "all") {
+ return buildVocab(getAll());
+ }
+ if (shouldCache && vocabCache.has(p)) {
+ return vocabCache.get(p) as string[];
+ }
+ const vocab = buildVocab(getCorpus(p));
+ if (shouldCache) vocabCache.set(p, vocab);
+ return vocab;
+}
+
function getAll(): IndexedDoc[] {
const home = indexDoc(homeDoc(), "product");
const corpora = CORPORA.filter((t) => t !== "product").flatMap(getCorpus);
@@ -88,6 +121,35 @@ function getAll(): IndexedDoc[] {
];
}
+type ScoredEntry = {
+ entry: IndexedDoc;
+ score: number;
+ needle: string | null;
+ full: boolean;
+};
+
+function runScoring(
+ entries: IndexedDoc[],
+ phrase: string,
+ usePhrase: boolean,
+ matchers: ReturnType<typeof makeMatcher>[],
+): ScoredEntry[] {
+ const scored: ScoredEntry[] = [];
+ for (const entry of entries) {
+ const { score, needle, full } = scoreDoc(
+ entry,
+ phrase,
+ usePhrase,
+ matchers,
+ );
+ if (score > 0) scored.push({ entry, score, needle, full });
+ }
+ scored.sort(
+ (a, b) => b.score - a.score || recency(b.entry) - recency(a.entry),
+ );
+ return scored;
+}
+
export function searchCorpus(params: {
p: Corpus | "all";
q: string | null | undefined;
@@ -96,9 +158,9 @@ export function searchCorpus(params: {
const entries = p === "all" ? getAll() : getCorpus(p);
const query = q?.trim() ?? "";
- const phrase = query.toLowerCase();
+ const phrase = normalizeForMatch(query);
// Drop 1-char terms — pure noise that word-boundary matching can't tame.
- const terms = phrase.split(/\s+/).filter((t) => t.length >= 2);
+ const terms = filterTerms(phrase.split(/\s+/).filter((t) => t.length >= 2));
if (!query) {
return entries
@@ -109,34 +171,33 @@ export function searchCorpus(params: {
const matchers = terms.map(makeMatcher);
const usePhrase = phrase.includes(" ");
- const scored: {
- entry: IndexedDoc;
- score: number;
- needle: string | null;
- full: boolean;
- }[] = [];
- for (const entry of entries) {
- const { score, needle, full } = scoreDoc(
- entry,
- phrase,
- usePhrase,
- matchers,
- );
- if (score > 0) scored.push({ entry, score, needle, full });
+ let scored = runScoring(entries, phrase, usePhrase, matchers);
+ let isFallback = false;
+
+ // Zero-result fallback: attempt per-term typo correction against corpus vocabulary.
+ if (scored.length === 0 && terms.length > 0) {
+ const vocab = getVocab(p);
+ const corrected = terms.map((t) => findCorrection(t, vocab) ?? t);
+ const anyFixed = corrected.some((c, i) => c !== terms[i]);
+ if (anyFixed) {
+ const corrPhrase = corrected.join(" ");
+ const corrUsePhrase = corrPhrase.includes(" ");
+ const corrMatchers = corrected.map(makeMatcher);
+ scored = runScoring(entries, corrPhrase, corrUsePhrase, corrMatchers);
+ isFallback = true;
+ }
}
- scored.sort(
- (a, b) => b.score - a.score || recency(b.entry) - recency(a.entry),
- );
-
return scored.map(({ entry, needle, full }) => {
+ // Corrected matches must never present as confident hits.
+ const tier = isFallback ? "partial" : full ? "full" : "partial";
let href = `${entry.doc.href}?q=${encodeURIComponent(query)}`;
if (needle) {
const slug = closestHeadingSlug(entry, needle);
if (slug) href = `${href}#${slug}`;
}
const snippet = getContentSnippet(entry.sanitized, needle);
- return toResult(entry, snippet, href, full ? "full" : "partial");
+ return toResult(entry, snippet, href, tier);
});
}
diff --git a/apps/web/src/content/utils/search-match.test.ts b/apps/web/src/content/utils/search-match.test.ts
index a5c689109..15e92226c 100644
--- a/apps/web/src/content/utils/search-match.test.ts
+++ b/apps/web/src/content/utils/search-match.test.ts
@@ -1,15 +1,28 @@
import { describe, expect, test } from "bun:test";
import {
+ WEIGHTS,
type Matcher,
+ buildHighlightRegex,
closestHeadingSlug,
+ filterTerms,
+ findCorrection,
getContentSnippet,
makeMatcher,
needleIndex,
+ normalizeForMatch,
sanitizeContent,
scoreDoc,
+ withinEditDistance1,
} from "./search-match";
+// .match() instead of .test() throughout — the returned regex is `g`-flagged
+// and .test would leak lastIndex between assertions.
+function marks(query: string, text: string): string[] {
+ const re = buildHighlightRegex(query);
+ return re ? (text.match(re) ?? []) : [];
+}
+
describe("makeMatcher", () => {
test("prefix match at a word start", () => {
expect(makeMatcher("monitor").test("synthetic monitoring")).toBe(true);
@@ -55,11 +68,13 @@ describe("needleIndex", () => {
function scorable(parts: {
title?: string;
headings?: string;
+ description?: string;
content?: string;
}) {
return {
titleLower: (parts.title ?? "").toLowerCase(),
headingsLower: (parts.headings ?? "").toLowerCase(),
+ descriptionLower: (parts.description ?? "").toLowerCase(),
sanitizedLower: (parts.content ?? "").toLowerCase(),
};
}
@@ -198,3 +213,258 @@ describe("sanitizeContent", () => {
);
});
});
+
+describe("foldTerm / makeMatcher plural folding", () => {
+ test("plural query matches singular doc — the verified bug", () => {
+ expect(makeMatcher("notifications").test("notification settings")).toBe(
+ true,
+ );
+ });
+
+ test("ies → y fold", () => {
+ expect(makeMatcher("dependencies").test("a dependency graph")).toBe(true);
+ });
+
+ test("s → singular fold", () => {
+ expect(makeMatcher("statuses").test("the status page")).toBe(true);
+ });
+
+ test("no over-folding of short / ss terms", () => {
+ // "ss" should not fold to "s"; "class" contains "ss" mid-word only
+ expect(makeMatcher("ss").test("a class act")).toBe(false);
+ });
+
+ test("needleIndex works for fold-matched single-word needles", () => {
+ expect(needleIndex("notification settings", "notifications")).toBe(0);
+ });
+});
+
+describe("filterTerms", () => {
+ test("removes stopwords from a mixed query", () => {
+ expect(filterTerms(["how", "to", "configure", "status"])).toEqual([
+ "configure",
+ "status",
+ ]);
+ });
+
+ test("returns all terms when every term is a stopword", () => {
+ expect(filterTerms(["how", "to"])).toEqual(["how", "to"]);
+ });
+});
+
+describe("scoreDoc stopword integration", () => {
+ test("stopword-heavy query gets allTermsBonus after filtering", () => {
+ const entry = scorable({
+ title: "status page subdomain",
+ content: "configure the subdomain for your status page",
+ });
+ const filtered = filterTerms([
+ "how",
+ "to",
+ "configure",
+ "status",
+ "page",
+ "subdomain",
+ ]);
+ const ms = matchers(...filtered);
+ const phrase = "how to configure status page subdomain";
+ const r = scoreDoc(entry, phrase, true, ms);
+ // Before the fix, "how" and "to" were in matchers but matched nothing,
+ // so allTermsBonus was never awarded. Now they're stripped and all 4
+ // content terms match, so allTermsBonus is included and score > 38.
+ expect(r.score).toBeGreaterThan(38);
+ });
+});
+
+describe("normalizeForMatch / hyphen phrase", () => {
+ test("hyphenated content matches space-separated query phrase", () => {
+ const normalized = normalizeForMatch("configure your status-page here");
+ const r = scoreDoc(
+ {
+ titleLower: "",
+ headingsLower: "",
+ descriptionLower: "",
+ sanitizedLower: normalized,
+ },
+ "status page",
+ true,
+ matchers("status", "page"),
+ );
+ expect(r.full).toBe(true);
+ expect(r.needle).toBe("status page");
+ });
+
+ test("getContentSnippet returns original text including hyphen", () => {
+ const snippet = getContentSnippet(
+ "configure your status-page here",
+ "status page",
+ );
+ expect(snippet).toContain("status-page");
+ });
+});
+
+describe("scoreDoc description field", () => {
+ test("phrase in description only scores phraseDescription and sets full, needle stays null", () => {
+ const r = scoreDoc(
+ scorable({ description: "uptime monitoring for your services" }),
+ "uptime monitoring",
+ true,
+ matchers("uptime", "monitoring"),
+ );
+ // phraseDescription (50) + termDescription for both terms + allTermsBonus
+ expect(r.score).toBeGreaterThanOrEqual(
+ WEIGHTS.phraseDescription +
+ WEIGHTS.termDescription * 2 +
+ WEIGHTS.allTermsBonus,
+ );
+ expect(r.full).toBe(true);
+ expect(r.needle).toBeNull();
+ });
+
+ test("term in description only contributes termDescription to score", () => {
+ const r = scoreDoc(
+ scorable({ description: "synthetic monitoring platform" }),
+ "synthetic",
+ false,
+ matchers("synthetic"),
+ );
+ expect(r.score).toBe(WEIGHTS.termDescription);
+ expect(r.needle).toBeNull();
+ });
+
+ test("two-term query: one term in description, other in content earns allTermsBonus", () => {
+ const r = scoreDoc(
+ scorable({
+ description: "synthetic checks",
+ content: "configure your monitoring setup",
+ }),
+ "synthetic monitoring",
+ true,
+ matchers("synthetic", "monitoring"),
+ );
+ expect(r.score).toBeGreaterThanOrEqual(
+ WEIGHTS.termDescription + WEIGHTS.termContent + WEIGHTS.allTermsBonus,
+ );
+ });
+
+ test("phrase-in-description doc outscores phrase-in-content doc", () => {
+ const descOnly = scoreDoc(
+ scorable({ description: "status page uptime" }),
+ "status page",
+ true,
+ matchers("status", "page"),
+ );
+ const contentOnly = scoreDoc(
+ scorable({ content: "status page uptime" }),
+ "status page",
+ true,
+ matchers("status", "page"),
+ );
+ expect(descOnly.score).toBeGreaterThan(contentOnly.score);
+ });
+});
+
+describe("withinEditDistance1", () => {
+ test("identical strings are within distance 1", () => {
+ expect(withinEditDistance1("monitor", "monitor")).toBe(true);
+ });
+
+ test("deletion: monitor → monitr", () => {
+ expect(withinEditDistance1("monitor", "monitr")).toBe(true);
+ });
+
+ test("adjacent transposition: monitor → montior", () => {
+ expect(withinEditDistance1("monitor", "montior")).toBe(true);
+ });
+
+ test("substitution: monitor → monidor", () => {
+ expect(withinEditDistance1("monitor", "monidor")).toBe(true);
+ });
+
+ test("more than one edit away returns false", () => {
+ expect(withinEditDistance1("monitor", "mon")).toBe(false);
+ });
+
+ test("two substitutions returns false", () => {
+ expect(withinEditDistance1("monitor", "mXnitYr")).toBe(false);
+ });
+});
+
+describe("findCorrection", () => {
+ test("corrects a deletion typo to the matching vocab word", () => {
+ expect(findCorrection("monitr", ["monitor", "status"])).toBe("monitor");
+ });
+
+ test("returns null for terms shorter than 4 characters", () => {
+ expect(findCorrection("xyz", ["xyz"])).toBeNull();
+ });
+
+ test("returns null when length diff is 2 (plurals owned by foldTerm)", () => {
+ expect(findCorrection("status", ["statuses"])).toBeNull();
+ });
+
+ test("different first letter never corrects", () => {
+ expect(findCorrection("ponitor", ["monitor"])).toBeNull();
+ });
+
+ test("deterministic tie-break: prefers lexicographically first equal-distance candidate", () => {
+ // "moniter" and "moniter" → only one real candidate, but two same-length options:
+ // "monitor" and "moniker" both start with 'm', same length, edit-distance 1
+ const result = findCorrection("moniter", ["moniker", "monitor"]);
+ expect(result).toBe("moniker");
+ });
+
+ test("returns null when no candidate is close enough", () => {
+ expect(findCorrection("xyzabc", ["monitor", "status", "probe"])).toBeNull();
+ });
+});
+
+describe("buildHighlightRegex", () => {
+ test("null for empty or whitespace-only queries", () => {
+ expect(buildHighlightRegex("")).toBeNull();
+ expect(buildHighlightRegex(" ")).toBeNull();
+ });
+
+ test("fold-matched text gets marked — singular doc, plural query", () => {
+ expect(marks("notifications", "up a notification channel")).toEqual([
+ "notification",
+ ]);
+ });
+
+ test("longest alternate wins — full plural marked where present", () => {
+ expect(marks("notifications", "receive notifications now")).toEqual([
+ "notifications",
+ ]);
+ });
+
+ test("all occurrences are marked, case-insensitively", () => {
+ expect(marks("monitor", "Monitor your monitors")).toEqual([
+ "Monitor",
+ "monitor",
+ ]);
+ });
+
+ test("contiguous phrase marks as one span", () => {
+ expect(marks("status page", "your status page here")).toEqual([
+ "status page",
+ ]);
+ });
+
+ test("stopwords are not marked on their own", () => {
+ expect(marks("how to configure", "walk to school")).toEqual([]);
+ expect(marks("how to configure", "then configure it")).toEqual([
+ "configure",
+ ]);
+ });
+
+ test("regex metacharacters in the query do not throw", () => {
+ expect(marks("c++", "learn c++ today")).toEqual(["c++"]);
+ });
+
+ test("hyphenated query still marks its terms", () => {
+ expect(marks("status-page", "the status page setup")).toEqual([
+ "status",
+ "page",
+ ]);
+ });
+});
diff --git a/apps/web/src/content/utils/search-match.ts b/apps/web/src/content/utils/search-match.ts
index 1a7ec59d9..f4747b8d7 100644
--- a/apps/web/src/content/utils/search-match.ts
+++ b/apps/web/src/content/utils/search-match.ts
@@ -4,9 +4,11 @@
export const WEIGHTS = {
phraseTitle: 100,
phraseHeading: 60,
+ phraseDescription: 50,
phraseContent: 30,
termTitle: 10,
termHeading: 6,
+ termDescription: 5,
termContent: 2,
allTermsBonus: 20,
} as const;
@@ -27,6 +29,27 @@ function isWordLed(term: string): boolean {
export type Matcher = { term: string; test: (text: string) => boolean };
+// Naive singular folds so a plural query still hits singular docs
+// ("notifications" → "notification", "dependencies" → "dependency").
+// Prefix matching already covers the singular→plural direction.
+export function foldTerm(term: string): string[] {
+ const out = [term];
+ if (term.endsWith("ies") && term.length >= 6)
+ out.push(`${term.slice(0, -3)}y`);
+ if (term.endsWith("es") && term.length >= 5) out.push(term.slice(0, -2));
+ if (term.endsWith("s") && !term.endsWith("ss") && term.length >= 4)
+ out.push(term.slice(0, -1));
+ return [...new Set(out)];
+}
+
+function termRegex(term: string): RegExp {
+ const alts = foldTerm(term)
+ .sort((a, b) => b.length - a.length)
+ .map(escapeRegExp)
+ .join("|");
+ return new RegExp(`\\b(?:${alts})`, "u");
+}
+
// Match at a word start, prefix allowed: "monitor" hits "monitoring", but "as" no
// longer hits "case". Note `\b` is ASCII-only even under the `u` flag, so boundary
// detection around accented letters is approximate — acceptable for our corpus.
@@ -34,28 +57,95 @@ export function makeMatcher(term: string): Matcher {
if (!isWordLed(term)) {
return { term, test: (text) => text.includes(term) };
}
- const re = new RegExp(`\\b${escapeRegExp(term)}`, "u");
+ const re = termRegex(term);
return { term, test: (text) => re.test(text) };
}
// Index of the first word-start occurrence (ASCII boundary), or the first substring
// occurrence for symbol-led needles. Both inputs must already be lowercased.
+// Multi-word needles (phrases) keep the plain escaped regex — folding a phrase would mangle it.
export function needleIndex(
haystackLower: string,
needleLower: string,
): number {
if (isWordLed(needleLower)) {
- const m = new RegExp(`\\b${escapeRegExp(needleLower)}`, "u").exec(
- haystackLower,
- );
+ const isSingleWord = !needleLower.includes(" ");
+ const re = isSingleWord
+ ? termRegex(needleLower)
+ : new RegExp(`\\b${escapeRegExp(needleLower)}`, "u");
+ const m = re.exec(haystackLower);
if (m) return m.index;
}
return haystackLower.indexOf(needleLower);
}
+const STOPWORDS = new Set([
+ "a",
+ "an",
+ "the",
+ "and",
+ "or",
+ "of",
+ "to",
+ "in",
+ "on",
+ "at",
+ "for",
+ "with",
+ "is",
+ "are",
+ "be",
+ "it",
+ "this",
+ "that",
+ "my",
+ "your",
+ "our",
+ "how",
+ "do",
+ "does",
+ "can",
+ "what",
+ "when",
+ "i",
+ "you",
+ "we",
+]);
+
+// Drop stopwords unless that empties the query (e.g. "how to") — better a
+// noisy match than none.
+export function filterTerms(terms: string[]): string[] {
+ const content = terms.filter((t) => !STOPWORDS.has(t));
+ return content.length > 0 ? content : terms;
+}
+
+// 1:1 char replacement — keeps every index aligned with the unnormalized
+// string, which snippet extraction depends on.
+export function normalizeForMatch(input: string): string {
+ return input.toLowerCase().replace(/[-_]/g, " ");
+}
+
+// Shared <mark> regex for search surfaces (cmdk snippets, ?q= page highlight) so
+// what gets marked mirrors what the server matched: contiguous phrase plus folded
+// content terms, longest alternate first, regex-escaped, global.
+export function buildHighlightRegex(query: string): RegExp | null {
+ const q = query.trim().toLowerCase();
+ if (!q) return null;
+ const terms = filterTerms(
+ normalizeForMatch(q)
+ .split(/\s+/)
+ .filter((t) => t.length >= 2),
+ ).flatMap(foldTerm);
+ const parts = [...new Set([q, ...terms])]
+ .sort((a, b) => b.length - a.length)
+ .map(escapeRegExp);
+ return new RegExp(parts.join("|"), "gi");
+}
+
type Scorable = {
titleLower: string;
headingsLower: string;
+ descriptionLower: string;
sanitizedLower: string;
};
@@ -80,6 +170,12 @@ export function scoreDoc(
score += WEIGHTS.phraseHeading;
phraseMatched = true;
}
+ if (entry.descriptionLower.includes(phrase)) {
+ score += WEIGHTS.phraseDescription;
+ phraseMatched = true;
+ // needle stays null — description is not part of sanitized, so snippet
+ // lookup would fail; the leading excerpt fallback is correct here.
+ }
if (entry.sanitizedLower.includes(phrase)) {
score += WEIGHTS.phraseContent;
phraseMatched = true;
@@ -91,14 +187,16 @@ export function scoreDoc(
for (const m of matchers) {
const inTitle = m.test(entry.titleLower);
const inHeading = m.test(entry.headingsLower);
+ const inDescription = m.test(entry.descriptionLower);
const inContent = m.test(entry.sanitizedLower);
if (inTitle) score += WEIGHTS.termTitle;
if (inHeading) score += WEIGHTS.termHeading;
+ if (inDescription) score += WEIGHTS.termDescription;
if (inContent) {
score += WEIGHTS.termContent;
if (!needle) needle = m.term;
}
- if (inTitle || inHeading || inContent) matched++;
+ if (inTitle || inHeading || inDescription || inContent) matched++;
}
if (matchers.length > 1 && matched === matchers.length)
@@ -134,7 +232,10 @@ export function getContentSnippet(
): string {
if (!needle) return `${sanitized.slice(0, 100)}...`;
- const matchIndex = needleIndex(sanitized.toLowerCase(), needle.toLowerCase());
+ const matchIndex = needleIndex(
+ normalizeForMatch(sanitized),
+ normalizeForMatch(needle),
+ );
if (matchIndex === -1) return `${sanitized.slice(0, 100)}...`;
let start = matchIndex;
@@ -161,6 +262,74 @@ export function getContentSnippet(
return snippet;
}
+// True when a and b are within Damerau-Levenshtein distance 1
+// (one insert, delete, substitute, or adjacent transposition).
+export function withinEditDistance1(a: string, b: string): boolean {
+ const diff = a.length - b.length;
+ if (diff > 1 || diff < -1) return false;
+ if (a === b) return true;
+
+ if (diff === 0) {
+ // Equal length: count mismatches; allow 1 substitution or 1 adjacent swap.
+ let first = -1;
+ let mismatches = 0;
+ for (let i = 0; i < a.length; i++) {
+ if (a[i] !== b[i]) {
+ mismatches++;
+ if (mismatches === 1) first = i;
+ if (mismatches > 2) return false;
+ }
+ }
+ if (mismatches <= 1) return true;
+ // Adjacent transposition: a[first] === b[first+1] && a[first+1] === b[first]
+ return (
+ first + 1 < a.length &&
+ a[first] === b[first + 1] &&
+ a[first + 1] === b[first]
+ );
+ }
+
+ // Length diff of 1: walk both, allow exactly one skip in the longer string.
+ const [longer, shorter] = diff > 0 ? [a, b] : [b, a];
+ let i = 0;
+ let j = 0;
+ let skips = 0;
+ while (i < longer.length && j < shorter.length) {
+ if (longer[i] !== shorter[j]) {
+ skips++;
+ if (skips > 1) return false;
+ i++;
+ } else {
+ i++;
+ j++;
+ }
+ }
+ return true;
+}
+
+// Cheapest plausible correction: same first letter, |len| ≤ 1, edit distance ≤ 1.
+// First-letter anchor kills most of the scan and most absurd corrections.
+export function findCorrection(term: string, vocab: string[]): string | null {
+ if (term.length < 4) return null;
+ let best: string | null = null;
+ let bestLenDiff = 2;
+ for (const word of vocab) {
+ if (word[0] !== term[0]) continue;
+ const lenDiff = Math.abs(word.length - term.length);
+ if (lenDiff > 1) continue;
+ if (!withinEditDistance1(term, word)) continue;
+ // Prefer smallest length difference, then lexicographically first.
+ if (
+ lenDiff < bestLenDiff ||
+ (lenDiff === bestLenDiff && best !== null && word < best)
+ ) {
+ best = word;
+ bestLenDiff = lenDiff;
+ }
+ }
+ return best;
+}
+
export function sanitizeContent(input: string) {
return stripTags(input)
.replace(/^#{1,6}\s+/gm, "") // strip markdown heading symbols, keep text