Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.41.16.0 — Conversation parser orchestrator.
*
* Drives the per-page parse pipeline:
*
* 1. Resolve date context from Page (D8 precedence:
* explicit > frontmatter.date > effective_date > 1970-01-01).
* 2. Score candidate patterns across the first N lines (D18). Pick
* the highest-scoring; tie-break on declared priority order.
* 3. Apply the winning pattern (multi-line continuations honored
* per D5).
* 4. If zero matched: optionally call LLM fallback (D15 opt-IN).
* 5. Optionally polish regex-matched output via LLM (D15 opt-IN).
* 6. Return ParseResult with phase + matched_pattern_id + diagnostics.
*
* Behavior contract: PR #1461's 33 tests (27 existing + 6 telegram-
* bracket) MUST pass against this orchestrator. The
* `parseConversation(body, opts)` shape from the prior
* `parseConversationMessages(body, opts)` is preserved via a thin
* adapter in `extract-conversation-facts.ts` (T5 retrofit).
*
* Pure-function inner core: no I/O, no LLM calls except via the
* fallback wrapper in `llm-fallback.ts` (T4). That wrapper is passed
* in via opts so this file stays test-isolatable.
*/
import {
BUILTIN_PATTERNS,
cleanSpeaker,
} from './builtins.ts';
import { blockNormalizer, normalizeBlockConversation } from './normalize-block.ts';
import { SCAN_LINE_BUFFER_BYTES } from '../page-body/limits.ts';
import type {
DateContext,
MatchedMessage,
ParseConversationOpts,
ParseResult,
PatternEntry,
} from './types.ts';
export type { ParseConversationOpts, ParseResult, MatchedMessage } from './types.ts';
/**
* How many head-of-body lines to score patterns against (D18).
* Higher = more accurate disambiguation, more regex calls per page.
* 10 balances both — typical chat exports have homogeneous shape so
* 10 lines is enough to differentiate Telegram from Discord from
* WhatsApp.
*/
const SCORING_HEAD_LINES = 10;
/**
* Head-pass score below which `parseConversation` falls back to a
* full-body re-score (v0.41.18+ fix for #1533 + Codex P1 #1).
*
* Why a threshold instead of `=== 0`: meeting pages start with
* `## Summary` + blockquotes + `## Transcript` headings. A
* blockquote like `> [12:00] Foo` can accidentally match an
* unrelated pattern's regex (irc-classic at score 0.1) which would
* suppress the fallback even when 175 of 226 lines further down are
* valid imessage-slack. 0.3 = "fewer than 3 of 10 head lines
* matched"; chat-only pages still score 1.0 and skip the fallback
* entirely, so the fast path is preserved.
*/
const SCORING_HEAD_TRIGGER_THRESHOLD = 0.3;
/**
* Minimum final winner score required to accept `regex_match`
* (v0.41.18+ Codex P1 #2). A 500-line essay with one stray
* `**Name** (date time):` line scores ~1/500 = 0.002 for
* imessage-slack, which without this floor would flip to
* `regex_match` with `messages.length = 1` — a false positive that
* silently corrupts downstream fact extraction. 0.05 = "at least 5%
* of non-blank lines anchored a message"; real transcript pages
* typically score 0.5+ and sail through, accidental anchors do not.
*/
const SCORING_MIN_ACCEPTANCE = 0.05;
/**
* Tie-breaker priority: lower index wins on score tie. Mirrors
* BUILTIN_PATTERNS declaration order. User-declared patterns get
* priority Infinity (lose every tie).
*/
function priorityOf(id: string): number {
const idx = BUILTIN_PATTERNS.findIndex((p) => p.id === id);
return idx >= 0 ? idx : Infinity;
}
/**
* Derive the date+timezone context for a Page per D8 precedence.
*
* 1. opts.fallbackDate (caller's explicit ISO date)
* 2. page.frontmatter.date — sliced to YYYY-MM-DD
* 3. page.effective_date — sliced to YYYY-MM-DD
* 4. '1970-01-01' epoch default
*
* Timezone comes from `page.frontmatter.timezone` (IANA tz like
* `'America/Los_Angeles'`) when present; otherwise undefined.
*
* Exported for tests + the eval CLI's debug command.
*/
export function deriveDateContext(opts: ParseConversationOpts): DateContext {
if (opts.fallbackDate) {
const sliced = opts.fallbackDate.slice(0, 10);
if (/^\d{4}-\d{2}-\d{2}$/.test(sliced)) {
return {
fallbackDate: sliced,
...extractTimezone(opts.page),
source: 'explicit',
};
}
}
const page = opts.page;
if (page?.frontmatter) {
const fmDate = page.frontmatter.date;
if (typeof fmDate === 'string') {
const sliced = fmDate.slice(0, 10);
if (/^\d{4}-\d{2}-\d{2}$/.test(sliced)) {
return {
fallbackDate: sliced,
...extractTimezone(page),
source: 'frontmatter_date',
};
}
}
}
if (page?.effective_date) {
return {
fallbackDate: page.effective_date.toISOString().slice(0, 10),
...extractTimezone(page),
source: 'effective_date',
};
}
return {
fallbackDate: '1970-01-01',
...extractTimezone(page),
source: 'epoch_default',
};
}
function extractTimezone(page: ParseConversationOpts['page']): Pick<DateContext, 'timezone' | 'invalid_timezone'> {
const tz = page?.frontmatter?.timezone;
if (typeof tz !== 'string' || tz.length === 0) return {};
return isValidTimeZone(tz) ? { timezone: tz } : { invalid_timezone: tz };
}
const timeZoneFormatters = new Map<string, Intl.DateTimeFormat>();
function timeZoneFormatter(tz: string): Intl.DateTimeFormat {
let formatter = timeZoneFormatters.get(tz);
if (!formatter) {
formatter = new Intl.DateTimeFormat('en-US', { timeZone: tz, timeZoneName: 'longOffset', year: 'numeric' });
timeZoneFormatters.set(tz, formatter);
}
return formatter;
}
/** Whether `tz` is an IANA zone the runtime knows (`Intl` throws RangeError on an unknown one). */
export function isValidTimeZone(tz: string): boolean {
try { timeZoneFormatter(tz); return true; } catch { return false; }
}
/** The zone's UTC offset in minutes at `instantMs` (`GMT-07:00` -> -420; `GMT` -> 0). */
function offsetMinutesAt(instantMs: number, tz: string): number {
const name = timeZoneFormatter(tz).formatToParts(new Date(instantMs)).find(part => part.type === 'timeZoneName')?.value ?? 'GMT';
const m = /^GMT(?:([+-])(\d{1,2})(?::?(\d{2}))?)?$/.exec(name);
if (!m || !m[1]) return 0;
return (m[1] === '-' ? -1 : 1) * (Number(m[2]) * 60 + Number(m[3] ?? 0));
}
/**
* #5430 (W14 P1.9): the UTC instant of a wall-clock time in an IANA zone,
* as `YYYY-MM-DDTHH:MM:00Z`. Pure `Date.UTC` arithmetic plus `Intl` offsets,
* so the host timezone never enters. Two DST edge cases have a fixed policy:
* a FOLD (the wall time occurs twice, autumn) resolves to the first
* occurrence (the earlier instant, still on daylight time); a GAP (the wall
* time never occurs, spring) resolves with the offset in force before the
* transition, so the instant lands just after the gap (02:30 in a 02:00->03:00
* gap becomes the instant 03:30 names).
*/
export function localToUtcIso(date: string, hour: number, minute: number, tz: string): string {
const [y, mo, d] = date.split('-').map(Number);
const wall = Date.UTC(y, mo - 1, d, hour, minute);
const day = 24 * 60 * 60 * 1000;
const candidates = new Set<number>();
for (const probe of [wall - day, wall, wall + day]) {
const offset = offsetMinutesAt(probe, tz);
const instant = wall - offset * 60 * 1000;
if (offsetMinutesAt(instant, tz) === offset) candidates.add(instant);
}
const instant = candidates.size > 0
? Math.min(...candidates)
: wall - offsetMinutesAt(wall - day, tz) * 60 * 1000;
return new Date(instant).toISOString().replace(/\.\d{3}Z$/, 'Z');
}
/** A message timestamp: the wall time in the context's zone when one is declared, else the UTC reading. */
function stampIso(date: string, hour: number, minute: number, dateCtx: DateContext): string {
if (dateCtx.timezone) return localToUtcIso(date, hour, minute, dateCtx.timezone);
return `${date}T${String(hour).padStart(2, '0')}:${String(minute).padStart(2, '0')}:00Z`;
}
/**
* Map a 12-hour pattern to 24-hour using the AM/PM marker.
* 12 AM = 0, 12 PM = 12, 1..11 PM = 13..23.
*/
function to24h(hour: number, ampm: string | undefined): number {
if (!ampm) return hour;
const am = ampm.toUpperCase();
if (am === 'PM' && hour < 12) return hour + 12;
if (am === 'AM' && hour === 12) return 0;
return hour;
}
/**
* Build ISO timestamp for a regex match. Handles inline-date patterns
* (date in capture groups) AND time-only patterns (date from
* DateContext).
*
* Special-cases:
* - telegram-text-export: month-name + day + year + 12h_ampm
* reconstruction.
* - whatsapp-iso: dd/mm/yy reconstruction.
* - whatsapp-us / discord-export / teams-export: mm/dd/yy + 12h_ampm.
*
* Returns null if reconstruction fails (caller treats as orphan line).
*/
function buildIso(
match: RegExpExecArray,
entry: PatternEntry,
dateCtx: DateContext,
): string | null {
const { captures } = entry;
// Pattern-specific date reconstruction.
switch (entry.id) {
case 'telegram-text-export': {
// groups: 1=speaker, 2=monthName, 3=day, 4=year, 5=hour, 6=min, 7=sec, 8=ampm
const monthName = match[2];
const day = Number(match[3]);
const year = Number(match[4]);
const hourRaw = Number(match[5]);
const minute = Number(match[6]);
const ampm = match[8];
const month = monthNameToIndex(monthName);
if (month < 0 || !Number.isFinite(day) || !Number.isFinite(year)) return null;
const hour = to24h(hourRaw, ampm);
const date = `${year}-${String(month + 1).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
return stampIso(date, hour, minute, dateCtx);
}
case 'whatsapp-iso': {
// groups: 1=dd, 2=mm, 3=yy, 4=hh, 5=mm, 6=ss, 7=speaker, 8=text
const day = Number(match[1]);
const month = Number(match[2]);
const yearRaw = Number(match[3]);
const year = yearRaw < 100 ? 2000 + yearRaw : yearRaw;
const hour = Number(match[4]);
const minute = Number(match[5]);
if (![day, month, year, hour, minute].every(Number.isFinite)) return null;
const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
return stampIso(date, hour, minute, dateCtx);
}
case 'whatsapp-us': {
// groups: 1=mm, 2=dd, 3=yy, 4=hh, 5=mm, 6=ampm, 7=speaker, 8=text
const month = Number(match[1]);
const day = Number(match[2]);
const yearRaw = Number(match[3]);
const year = yearRaw < 100 ? 2000 + yearRaw : yearRaw;
const hourRaw = Number(match[4]);
const minute = Number(match[5]);
const ampm = match[6];
if (![month, day, year, hourRaw, minute].every(Number.isFinite)) return null;
const hour = to24h(hourRaw, ampm);
const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
return stampIso(date, hour, minute, dateCtx);
}
case 'discord-export': {
// groups: 1=mm, 2=dd, 3=yyyy, 4=hh, 5=mm, 6=ampm, 7=speaker
const month = Number(match[1]);
const day = Number(match[2]);
const year = Number(match[3]);
const hourRaw = Number(match[4]);
const minute = Number(match[5]);
const ampm = match[6];
if (![month, day, year, hourRaw, minute].every(Number.isFinite)) return null;
const hour = to24h(hourRaw, ampm);
const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
return stampIso(date, hour, minute, dateCtx);
}
case 'teams-export': {
// groups: 1=speaker, 2=mm, 3=dd, 4=yyyy, 5=hh, 6=mm, 7=ampm, 8=text
const month = Number(match[2]);
const day = Number(match[3]);
const year = Number(match[4]);
const hourRaw = Number(match[5]);
const minute = Number(match[6]);
const ampm = match[7];
if (![month, day, year, hourRaw, minute].every(Number.isFinite)) return null;
const hour = to24h(hourRaw, ampm);
const date = `${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`;
return stampIso(date, hour, minute, dateCtx);
}
}
// Generic path for patterns whose captures map directly to
// date/hour/minute/ampm groups.
if (
entry.date_source === 'inline' &&
captures.date_group !== undefined &&
captures.hour_group !== undefined &&
captures.minute_group !== undefined
) {
const date = match[captures.date_group];
if (typeof date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(date)) {
return null;
}
const hourRaw = Number(match[captures.hour_group]);
const minute = Number(match[captures.minute_group]);
if (!Number.isFinite(hourRaw) || !Number.isFinite(minute)) return null;
const ampm =
captures.ampm_group !== undefined ? match[captures.ampm_group] : undefined;
const hour = to24h(hourRaw, ampm);
return stampIso(date, hour, minute, dateCtx);
}
// Time-only patterns: date from DateContext.
if (
entry.date_source === 'frontmatter' &&
captures.hour_group !== undefined &&
captures.minute_group !== undefined
) {
const hourRaw = Number(match[captures.hour_group]);
const minute = Number(match[captures.minute_group]);
if (!Number.isFinite(hourRaw) || !Number.isFinite(minute)) return null;
const ampm =
captures.ampm_group !== undefined ? match[captures.ampm_group] : undefined;
const hour = to24h(hourRaw, ampm);
return stampIso(dateCtx.fallbackDate, hour, minute, dateCtx);
}
// No-time patterns (irc-classic): only frontmatter date is
// available; anchor every message at 00:00:00 of that day.
// Honest: messages lose intra-day ordering, but at least they
// parse and the day-level fact attribution is correct.
if (entry.date_source === 'frontmatter' && captures.hour_group === undefined) {
return stampIso(dateCtx.fallbackDate, 0, 0, dateCtx);
}
return null;
}
const MONTHS_SHORT = [
'jan', 'feb', 'mar', 'apr', 'may', 'jun',
'jul', 'aug', 'sep', 'oct', 'nov', 'dec',
];
function monthNameToIndex(name: string): number {
return MONTHS_SHORT.indexOf(name.toLowerCase().slice(0, 3));
}
// #4136 — folded-heading detection constants. The label capture is capped at
// 48 chars (privacy: labels can carry page text into stderr/JSON) and the
// list at 10 entries per parse.
/** A markdown day heading (`## 2024-03-15`, levels 1-4): the date context for time-only patterns. */
export const DAY_HEADER_RE = /^#{1,4}\s+(\d{4}-\d{2}-\d{2})\s*$/;
const UNRECOGNIZED_HEADING_RE = /^(#{1,6})\s+(.{1,48}?)\s*:?\s*$/;
const FENCE_RE = /^(```|~~~)/;
const MAX_UNRECOGNIZED_HEADINGS = 10;
/**
* Apply ONE pattern to the full body. Returns the matched messages
* with their ISO timestamps. Handles multi-line continuations per D5.
*
* For multi_line=true patterns: the regex matches ONLY the FIRST line
* of a multi-line message; subsequent lines until the next anchor
* (next match or end-of-body) become the body.
*
* For multi_line=false patterns: the regex matches a complete line;
* continuation lines (orphan lines after a match) append to the
* previous message's body.
*
* Exported for unit tests + the eval CLI debug command.
*/
export function applyPattern(
body: string,
entry: PatternEntry,
dateCtx: DateContext,
diag?: { unrecognized_headings: string[]; date_fallback_count?: number },
): MatchedMessage[] {
if (!body) return [];
const out: MatchedMessage[] = [];
const lines = body.split(/\r?\n/);
const allowedAnchors = entry.id === 'python-dict-utterance'
? speakerObjectAnchorsAllowed(body, entry)
: undefined;
// Some multi-day conversation exports use markdown date headings instead
// of repeating a date on every message. Keep the caller's context immutable
// while advancing a local date anchor as those headings are encountered.
const runningCtx: DateContext = { ...dateCtx };
const dateHeaderRe = DAY_HEADER_RE;
// #4136 — when the WINNING pattern anchors on headings, a heading-shaped
// line whose label is outside the pattern's closed speaker set is not
// rejected and not reported: it folds into the PREVIOUS turn's body
// (heading line and all), or drops silently before the first anchor.
// The parse still returns phase 'regex_match', so one speaker gets
// credited with another's words and nothing downstream declines. Detection
// is diagnostic-only (zero behavior change here): collect the folded
// labels so ParseResult can surface them. Fence-aware so a transcript
// whose answers paste markdown/shell inside code fences is not flagged.
// A heading anchor that carries its own inline date (email-thread-heading)
// cannot be mistaken for a section title, so its other headings are body
// text, not lost speakers.
const headingAnchored =
diag !== undefined &&
entry.multi_line === true &&
entry.score_continuations_as_body === true &&
entry.captures.date_group === undefined &&
(entry.test_positive ?? []).some((s) => /^#{2,3}\s/.test(s));
let fenceMarker: '```' | '~~~' | null = null;
const collectFoldedHeading = (line: string): void => {
if (!headingAnchored || fenceMarker !== null || !diag) return;
if (diag.unrecognized_headings.length >= MAX_UNRECOGNIZED_HEADINGS) return;
const h = UNRECOGNIZED_HEADING_RE.exec(line);
if (!h) return;
const label = h[2].trim();
// Speaker-shaped-ish cap: ≤3 whitespace tokens, no sentence punctuation
// tail — long prose headings are section titles, not lost speakers.
if (!label || label.split(/\s+/).length > 3 || /[.!?]$/.test(label)) return;
if (entry.regex.test(line)) return; // a real anchor, not a fold
if (!diag.unrecognized_headings.includes(label)) diag.unrecognized_headings.push(label);
};
for (let i = 0; i < lines.length; i++) {
const rawLine = lines[i];
const line = rawLine.trim();
if (!line) continue;
if (headingAnchored) {
// Adversarial F6 (partial): a fence closes only on ITS OWN marker —
// CommonMark treats a mismatched marker as content, so `~~~` must not
// close a ```-opened fence. (An UNCLOSED fence still suppresses
// detection for the rest of the document — CommonMark-consistent, and
// the fail direction is warn-noise-free but detection-free; noted in
// the PR body as a known residual for truncated-LLM-output corpora.)
const fm = FENCE_RE.exec(line)?.[1] as '```' | '~~~' | undefined;
if (fm) {
if (fenceMarker === null) fenceMarker = fm;
else if (fenceMarker === fm) fenceMarker = null;
}
}
if (allowedAnchors?.[i] === false) continue;
const dateHeader = dateHeaderRe.exec(line);
if (dateHeader) {
runningCtx.fallbackDate = dateHeader[1];
continue;
}
if (allowedAnchors && !entry.regex.test(line)) continue;
// Quick-reject fast path.
if (entry.quick_reject && !entry.quick_reject.test(line)) {
collectFoldedHeading(line); // #4136 — folded below or dropped pre-anchor
// Continuation handling for orphan lines.
if (out.length > 0) {
out[out.length - 1].text = out[out.length - 1].text
? `${out[out.length - 1].text}\n${line}`
: line;
}
continue;
}
const m = entry.regex.exec(line);
if (m) {
let iso = buildIso(m, entry, runningCtx);
if (iso === null) {
// Reconstruction failed (e.g. a localized month name). Dropping the
// anchor would fold this message's body into the PREVIOUS speaker
// (silent misattribution, phase still regex_match), so open the
// message anyway and count it. Inherit the previous anchor's
// timestamp so ordering and downstream segment continuity survive;
// midnight of the page date only when this is the first anchor.
iso = out[out.length - 1]?.timestamp ?? stampIso(runningCtx.fallbackDate, 0, 0, runningCtx);
if (diag) diag.date_fallback_count = (diag.date_fallback_count ?? 0) + 1;
}
const rawSpeaker = m[entry.captures.speaker_group] ?? '';
const speaker = cleanSpeaker(rawSpeaker, entry.speaker_clean);
let text = '';
if (entry.captures.text_group > 0) {
text = (m[entry.captures.text_group] ?? '').trim();
}
// Multi-line patterns: text on next line(s) until next anchor.
// (Even when text_group is set, multi_line=true means SUBSEQUENT
// non-anchor lines also absorb into this message's body.)
out.push({ speaker, timestamp: iso, text });
} else {
// Passed quick_reject but failed the full regex (e.g. '## Assistant
// Bot' against the closed-set heading pattern) — the OTHER fold site.
collectFoldedHeading(line); // #4136
if (out.length > 0) {
// Continuation line.
out[out.length - 1].text = out[out.length - 1].text
? `${out[out.length - 1].text}\n${line}`
: line;
}
}
}
return out;
}
/**
* Split a body into non-blank trimmed lines. `headCap` (when set)
* limits the result to the first N lines — used by the head-pass
* scorer; omit for full-body scoring.
*/
function getNonBlankLines(body: string, headCap?: number): string[] {
if (!body) return [];
const all = body
.split(/\r?\n/)
.map((l) => l.trim())
.filter((l) => l.length > 0);
return headCap !== undefined ? all.slice(0, headCap) : all;
}
/** speakerObjectAnchorsAllowed one line at a time: line(raw) says whether that line may anchor; `invalid` voids every line. */
function speakerObjectScanner(entry: PatternEntry): { line(raw: string): boolean; readonly invalid: boolean } {
let fence: string | undefined;
let inTranscript: boolean | undefined;
let invalidRegion = false;
let malformedPreamble = false;
let preambleQuote: string | undefined;
const preambleClosers: string[] = [];
const scanPreamble = (line: string): void => {
for (let i = 0; i < line.length; i++) {
const char = line[i];
if (preambleQuote) {
if (char === '\\') i++;
else if (line.startsWith(preambleQuote, i)) {
i += preambleQuote.length - 1;
preambleQuote = undefined;
}
continue;
}
if (char === "'" || char === '"' || char === '`') {
let wordStart = i;
while (wordStart > 0 && /[\p{L}\p{N}_]/u.test(line[wordStart - 1])) wordStart--;
const word = line.slice(wordStart, i);
if (char === "'" && word && !/^(?:r|u|b|f|fr|rf|br|rb)$/i.test(word)) continue;
preambleQuote = line.startsWith(char.repeat(3), i) ? char.repeat(3) : char;
i += preambleQuote.length - 1;
} else if (char === '{') preambleClosers.push('}');
else if (char === '[') preambleClosers.push(']');
else if (char === '(') preambleClosers.push(')');
else if ('}])'.includes(char) && preambleClosers.length > 0 && preambleClosers.pop() !== char) malformedPreamble = true;
}
};
return {
get invalid() { return invalidRegion; },
line: (raw: string): boolean => {
const line = raw.trim();
if (malformedPreamble || preambleQuote || preambleClosers.length > 0) {
if (entry.regex.test(line) || /^#{1,6}\s+Transcript\s*$/i.test(line)) invalidRegion = true;
if (!malformedPreamble) scanPreamble(line);
return false;
}
const marker = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(raw);
if (fence) {
if (marker && marker[1][0] === fence[0] && marker[1].length >= fence.length && !marker[2].trim()) fence = undefined;
return false;
}
if (marker) {
fence = marker[1];
return false;
}
if (!line || /^(?: {4}| *\t)/.test(raw)) return false;
if (/^#{1,6}\s+Transcript\s*$/i.test(line)) {
inTranscript = true;
return false;
}
if (/^#{1,4}\s+\d{4}-\d{2}-\d{2}\s*$/.test(line)) return inTranscript !== false;
if (/^#{1,6}\s/.test(line)) {
inTranscript = false;
scanPreamble(line);
return false;
}
const matches = entry.regex.test(line);
if (inTranscript === undefined) {
inTranscript = matches;
if (!matches) invalidRegion = true;
} else if (inTranscript && !matches) {
invalidRegion = true;
} else if (!inTranscript) {
scanPreamble(line);
}
return inTranscript && matches;
},
};
}
function speakerObjectAnchorsAllowed(body: string, entry: PatternEntry): boolean[] {
const scanner = speakerObjectScanner(entry);
const allowed = body.split(/\r?\n/).map(raw => scanner.line(raw));
return scanner.invalid ? allowed.fill(false) : allowed;
}
function speakerObjectScoringLines(body: string, entry: PatternEntry, headCap?: number): string[] {
const allowed = speakerObjectAnchorsAllowed(body, entry);
const lines = body.split(/\r?\n/)
.map((line, index) => ({ line: line.trim(), allowed: allowed[index] }))
.filter(({ line }) => line.length > 0)
.map(({ line, allowed }) => allowed ? line : '');
return headCap === undefined ? lines : lines.slice(0, headCap);
}
/**
* Core scorer over a pre-split line array. Both `scorePattern` (head
* window) and `scorePatternFull` (whole body) delegate here so the
* quick_reject + regex loop lives in one place. Reused by
* `parseConversation`'s fallback path shares a pre-split array across
* candidates that use ordinary line scoring.
*/
function scoreFromLines(
lines: readonly string[],
entry: PatternEntry,
): number {
const score = lineScore(entry);
for (const line of lines) scoreLine(score, entry, line);
return scoreValue(score, entry);
}
/** scoreFromLines' counters over the lines scored so far, so a body can be scored one line at a time. */
interface LineScore { lines: number; anchored: number; candidates: number; firstLineAnchored: boolean; firstAnchorIndex: number; speakers: Set<string> | undefined }
function lineScore(entry: PatternEntry): LineScore {
// Only populated when score_continuations_min_distinct_speakers is set
// (avoids a Set + exec() per line for every other pattern, which only
// needs the boolean match `test()` already gave before this change).
return { lines: 0, anchored: 0, candidates: 0, firstLineAnchored: false, firstAnchorIndex: -1,
speakers: entry.score_continuations_min_distinct_speakers !== undefined ? new Set() : undefined };
}
function scoreLine(score: LineScore, entry: PatternEntry, line: string): void {
const index = score.lines++;
if (entry.quick_reject && !entry.quick_reject.test(line)) return;
score.candidates++;
let isMatch: boolean;
if (score.speakers) {
const m = entry.regex.exec(line);
isMatch = m !== null;
if (m) {
const speaker = m[entry.captures.speaker_group];
if (speaker) score.speakers.add(speaker);
}
} else {
isMatch = entry.regex.test(line);
}
if (isMatch) {
score.anchored++;
if (index === 0) score.firstLineAnchored = true;
if (score.firstAnchorIndex === -1) score.firstAnchorIndex = index;
}
}
function scoreValue(score: LineScore, entry: PatternEntry): number {
if (score.lines === 0) return 0;
const distinctSpeakersOk =
entry.score_continuations_min_distinct_speakers === undefined ||
(score.speakers?.size ?? 0) >=
entry.score_continuations_min_distinct_speakers;
// Bounds how far into the body the FIRST anchor may appear before the
// candidate-only density score activates. A genuine export's anchor
// grammar starts near the top of the body (allowing a short title/heading
// preamble); an anchor pair merely embedded deep inside an unrelated long
// document — which would otherwise get the SAME density immunity once
// both roles are present — sits far past this bound instead.
const preambleOk =
entry.score_continuations_max_preamble_lines === undefined ||
(score.firstAnchorIndex !== -1 &&
score.firstAnchorIndex <= entry.score_continuations_max_preamble_lines);
if (
entry.score_continuations_as_body &&
entry.multi_line &&
entry.quick_reject &&
score.candidates > 0 &&
(score.anchored >= 2 || score.firstLineAnchored) &&
distinctSpeakersOk &&
preambleOk
) {
return score.anchored / score.candidates;
}
return score.anchored / score.lines;
}
/**
* Score how well a pattern matches the first N lines of a body (D18).
* Returns 0..1 ratio of matched lines. Higher = more confident.
*
* Quick_reject is honored. Patterns that opt into
* `score_continuations_as_body` may exclude continuation lines from the
* denominator only after the scorer sees two anchors, or an anchor on the
* first non-blank line. Otherwise the ordinary full-body density applies.
*
* Exported for tests.
*/
export function scorePattern(body: string, entry: PatternEntry): number {
if (entry.id === 'python-dict-utterance') return scoreFromLines(speakerObjectScoringLines(body, entry, SCORING_HEAD_LINES), entry);
return scoreFromLines(getNonBlankLines(body, SCORING_HEAD_LINES), entry);
}
/**
* Score how well a pattern matches the FULL body, no head cap
* (v0.41.18+ Codex P1 #1 — full-body fallback when head pass falls
* below `SCORING_HEAD_TRIGGER_THRESHOLD`).
*
* Cost-aware: in `parseConversation`'s fallback path we pre-split
* ONCE and route through `scoreFromLines` directly, NOT through this
* wrapper, to avoid 12 redundant body splits per pass. This wrapper
* exists for direct unit testing and for any future caller that
* needs full-body scoring of a single pattern.
*/
export function scorePatternFull(body: string, entry: PatternEntry): number {
if (entry.id === 'python-dict-utterance') return scoreFromLines(speakerObjectScoringLines(body, entry), entry);
return scoreFromLines(getNonBlankLines(body), entry);
}
type Scored = { entry: PatternEntry; score: number; priority: number };
function sortScored(arr: Scored[]): Scored[] {
return arr.sort((a, b) => {
if (b.score !== a.score) return b.score - a.score;
return a.priority - b.priority;
});
}
/**
* D18: score every candidate on the head; highest first. Tie-break on
* declared priority order (built-in declaration order; user patterns
* lose every tie).
*/
function headScores(body: string, candidates: readonly PatternEntry[]): Scored[] {
return sortScored(candidates.map((entry) => ({
entry,
score: scorePattern(body, entry),
priority: priorityOf(entry.id),
})));
}
/**
* The built-in pattern a page's head picks by parseConversation's head
* pass, or null when that pass would not decide alone: the best head
* score is below the full-body fallback threshold, or the winner needs a
* full-body re-score. Segmented pages choose their message-anchor cut
* pattern with it once, from a bounded head (page-body/segmenter.ts).
*/
export function pickHeadPattern(head: string): string | null {
if (!head) return null;
const top = headScores(head, BUILTIN_PATTERNS)[0];
if (!top || top.score < SCORING_HEAD_TRIGGER_THRESHOLD || top.entry.score_full_body) return null;
return top.entry.id;
}
/**
* parseConversation's pattern choice over a body fed in pieces (a segmented page's occurrences, in order), holding
* counters instead of the body: the same block-format collapse (when any block header appears), head pass over the first
* SCORING_HEAD_LINES non-blank lines, full-body re-score, acceptance floor and structure gate, over the built-in
* patterns. `finish()` returns the pattern id parseConversation(body).matched_pattern_id would, or null. A line over
* SCAN_LINE_BUFFER_BYTES characters is scored by its first that many characters (every pattern anchors at the line's
* start) and the rest is never held (a block message's body is held up to that many characters), so the detector retains
* at most that much text between pieces.
*/
export function createPatternDetector(): { push(text: string): void; finish(): string | null } {
const raw = scoringTrack();
const block = scoringTrack();
const normalizer = blockNormalizer(line => block.line(line), { maxBodyChars: SCAN_LINE_BUFFER_BYTES });
let partial = '';
let oversized: string | null = null;
const line = (text: string) => {
const scored = text.length > SCAN_LINE_BUFFER_BYTES ? text.slice(0, SCAN_LINE_BUFFER_BYTES) : text;
normalizer.line(scored);
raw.line(scored.endsWith('\r') ? scored.slice(0, -1) : scored);
};
return {
push(text: string) {
let rest = text;
if (oversized !== null) {
const end = rest.indexOf('\n');
if (end < 0) return;
line(oversized);
oversized = null;
rest = rest.slice(end + 1);
}
const lines = (partial + rest).split('\n');
partial = lines.pop()!;
for (const l of lines) line(l);
if (partial.length > SCAN_LINE_BUFFER_BYTES) { oversized = Buffer.from(partial.slice(0, SCAN_LINE_BUFFER_BYTES)).toString(); partial = ''; }
},
finish() {
line(oversized ?? partial);
partial = '';
oversized = null;
normalizer.end();
return (normalizer.headers > 0 ? block : raw).finish();
},
};
}
/** One normalized body's scores, fed line by line (createPatternDetector). */
function scoringTrack(): { line(raw: string): void; finish(): string | null } {
const speakerObject = BUILTIN_PATTERNS.find((p) => p.id === 'python-dict-utterance');
const scanner = speakerObject ? speakerObjectScanner(speakerObject) : null;
const head = BUILTIN_PATTERNS.map((entry) => lineScore(entry));
const full = BUILTIN_PATTERNS.map((entry) => lineScore(entry));
// The structure gate's input: each anchored message's speaker, until one repeats.
const gates = BUILTIN_PATTERNS.map((entry) => (entry.repeat_speaker_min_turns === undefined ? null : { turns: 0, speakers: new Set<string>(), repeated: false }));
let nonBlank = 0;
return {
line(raw: string) {
const allowed = scanner ? scanner.line(raw) : true;
const line = raw.trim();
if (!line) return;
nonBlank++;
BUILTIN_PATTERNS.forEach((entry, i) => {
const scored = entry === speakerObject && !allowed ? '' : line;
if (nonBlank <= SCORING_HEAD_LINES) scoreLine(head[i]!, entry, scored);
scoreLine(full[i]!, entry, scored);
const gate = gates[i];
if (!gate || gate.repeated || DAY_HEADER_RE.test(line) || (entry.quick_reject && !entry.quick_reject.test(line))) return;
const m = entry.regex.exec(line);
if (!m) return;
gate.turns++;
const speaker = cleanSpeaker(m[entry.captures.speaker_group] ?? '', entry.speaker_clean);
if (gate.speakers.has(speaker)) { gate.repeated = true; gate.speakers.clear(); } else gate.speakers.add(speaker);
});
},
finish() {
const value = (scores: LineScore[], i: number) => (scanner?.invalid && BUILTIN_PATTERNS[i] === speakerObject ? 0 : scoreValue(scores[i]!, BUILTIN_PATTERNS[i]!));
const ranked = (scores: LineScore[]) => sortScored(BUILTIN_PATTERNS.map((entry, i) => ({ entry, score: value(scores, i), priority: priorityOf(entry.id) })));
let scored = ranked(head);
let fullBodyScored = false;
if (scored[0]!.score < SCORING_HEAD_TRIGGER_THRESHOLD) {
scored = ranked(full);
fullBodyScored = true;
}
const top = scored[0]!;
const i = BUILTIN_PATTERNS.indexOf(top.entry);
const score = top.entry.score_full_body && !fullBodyScored ? value(full, i) : top.score;
if (score < SCORING_MIN_ACCEPTANCE) return null;
const gate = gates[i];
if (gate && !gate.repeated && gate.turns >= top.entry.repeat_speaker_min_turns!) return null;
return top.entry.id;
},
};
}
/**
* Whether `line` opens a message under built-in pattern `patternId`, as
* applyPattern reads it (trimmed; quick_reject, then the anchor regex), and
* the inline date it carries when the pattern has one. Null when it does not
* open one (day headers never do) or the id is unknown.
*/
export function matchAnchorLine(patternId: string, line: string): { date: string | null } | null {
const entry = BUILTIN_PATTERNS.find((p) => p.id === patternId);
const text = line.trim();
if (!entry || !text) return null;
if (entry.quick_reject && !entry.quick_reject.test(text)) return null;
const m = entry.regex.exec(text);
if (!m) return null;
const group = entry.captures.date_group;
return { date: group === undefined ? null : m[group] ?? null };
}
export function isAnchorLine(patternId: string, line: string): boolean {
return matchAnchorLine(patternId, line) !== null;
}
/**
* Parse a conversation body into messages. Tries built-in patterns
* (minus disabled), then user patterns (D7-validated), then optional
* LLM fallback (T4; not yet wired).
*
* The shape matches PR #1461's `parseConversationMessages(body, opts)`
* for back-compat. `extract-conversation-facts.ts` adapts via
* `parseConversation(body, { page })` in T5.
*/
export function parseConversation(
body: string,
opts: ParseConversationOpts = {},
): ParseResult {
if (!body) {
return { messages: [], phase: 'no_match' };
}
// Pre-pass: collapse block-format chat exports (header + indented body, e.g.
// the Slack collector's `- **Name** (Mon 11:18)\n body…`) into the canonical
// single-line shape the built-in patterns recognize. Strict no-op when no
// block header is present, so already-canonical content is untouched.
body = normalizeBlockConversation(body);
const dateCtx = deriveDateContext(opts);
// Assemble candidate pool: built-ins (minus disabled) + user patterns.
const disabledSet = new Set(opts.disabledBuiltinIds ?? []);
const builtinPool = BUILTIN_PATTERNS.filter((p) => !disabledSet.has(p.id));
const userPool = opts.userPatterns ?? [];
const candidates: readonly PatternEntry[] = [...builtinPool, ...userPool];
if (candidates.length === 0) {
return { messages: [], phase: 'no_match' };
}
let scored = headScores(body, candidates);
// REGRESSION (closes #1533 + Codex P1 #1): meeting pages have
// ## Summary + blockquote + ## Transcript ahead of the chat. The
// pre-fix "trigger fallback only when score === 0" shape left a
// real bug class open — a stray head match (e.g. a blockquote
// that accidentally matches an unrelated pattern at 0.1)
// suppressed the fallback even when 175 of 226 lines further down
// were valid imessage-slack. Threshold 0.3 means "fewer than 3 of
// 10 head lines matched"; chat-only pages still score 1.0 and skip
// the fallback. Re-score every candidate against the full body,
// pre-splitting ONCE to avoid 12 redundant body splits.
let fullBodyScored = false;
if (scored[0].score < SCORING_HEAD_TRIGGER_THRESHOLD) {
const allLines = getNonBlankLines(body);
scored = candidates.map((entry) => ({
entry,
score: entry.id === 'python-dict-utterance' ? scorePatternFull(body, entry) : scoreFromLines(allLines, entry),
priority: priorityOf(entry.id),
}));
sortScored(scored);
fullBodyScored = true;
// NOTE: patterns_scored stays as scored.length (= candidate
// count, typically 12) even when the fallback runs — the
// diagnostic reports "candidates considered" not "scoring
// attempts" (Codex P2 #7).
}
const top = scored[0];
const patternsScored = scored.length;
// v0.41.29.0 (Codex F1): broad no-time patterns (`bold-name-no-time`,
// regex matches any `**Label:** text`) can win the HEAD pass on a
// prose notes page whose first 10 lines happen to hold 3+ bold labels
// (head score 0.3, NOT < SCORING_HEAD_TRIGGER_THRESHOLD, so the
// full-body fallback above never ran). 0.3 clears the 0.05 floor and
// the page mis-parses as a conversation. When the winner is flagged
// `score_full_body` and we have NOT already scored full-body, recompute
// its score over the whole document so the floor judges true density.
// We do NOT re-pick the winner: the head pick is already the best
// candidate; this only tightens its acceptance (a real transcript
// stays ~1.0; a 3/200-label notes page drops to ~0.015 → no_match).
if (top.entry.score_full_body && !fullBodyScored) {
top.score = scorePatternFull(body, top.entry);
}
// Minimum acceptance floor (closes Codex P1 #2): an essay with
// one stray `**Name** (date time):` line scores ~1/300 ≈ 0.003 —
// below the 5% floor we stay no_match instead of returning a
// 1-message false positive. Real transcript pages typically score
// 0.5+ and sail through.
if (top.score < SCORING_MIN_ACCEPTANCE) {
return {
messages: [],
phase: 'no_match',
patterns_scored: patternsScored,
unmatched_line_count: opts.diagnostic
? body.split(/\r?\n/).filter((l) => l.trim().length > 0).length
: undefined,
};
}
const diag = { unrecognized_headings: [] as string[], date_fallback_count: 0 };
const messages = applyPattern(body, top.entry, dateCtx, diag);
// Structure gate (N12-1): enough turns with every speaker distinct is a
// list of labels, not a conversation.
const minTurns = top.entry.repeat_speaker_min_turns;
if (
minTurns !== undefined &&
messages.length >= minTurns &&
new Set(messages.map((m) => m.speaker)).size === messages.length
) {
return {
messages: [],
phase: 'no_match',
patterns_scored: patternsScored,
unmatched_line_count: opts.diagnostic
? body.split(/\r?\n/).filter((l) => l.trim().length > 0).length
: undefined,
};
}
// Timezone warning surface (D19).
let timezone_warning: string | undefined;
if (dateCtx.invalid_timezone && messages.length > 0) {
// #5430: a declared zone the runtime does not know keeps UTC, and says so.
timezone_warning = `[conversation-parser] pattern=${top.entry.id} frontmatter timezone '${dateCtx.invalid_timezone}' is not a valid IANA zone; assumed UTC for timestamps. Fix 'timezone:' in the page frontmatter (for example America/Los_Angeles) for accurate facts`;
} else if (
top.entry.timezone_policy === 'utc_assumed_with_warn' &&
!dateCtx.timezone &&
messages.length > 0
) {
timezone_warning = `[conversation-parser] pattern=${top.entry.id} assumed UTC for time-only timestamps; add 'timezone: <IANA>' to page frontmatter for accurate facts`;
}
return {
messages,
phase: 'regex_match',
matched_pattern_id: top.entry.id,
patterns_scored: patternsScored,
timezone_warning,
// Anchors rescued onto a fallback timestamp; undefined when zero.
date_fallback_count: diag.date_fallback_count || undefined,
// #4136 — populated unconditionally (NOT behind opts.diagnostic): the
// extractor's decline gate depends on it. Undefined when empty.
unrecognized_headings:
diag.unrecognized_headings.length > 0 ? diag.unrecognized_headings : undefined,
unmatched_line_count: opts.diagnostic
? body
.split(/\r?\n/)
.filter((l) => l.trim().length > 0).length - messages.length
: undefined,
};
}