Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust
47.7 KB
Raw Blame
//! FTS5 `MATCH` query preparation for user/agent-supplied search text.
//!
//! FTS5 treats `column:term` as a column-qualified search. Natural-language
//! queries that contain bare colons (`pick: handoff`, `memory: bootstrap`) make
//! SQLite error with `no such column: pick` because only `title` and `body`
//! exist on the FTS tables. Unknown bare column syntax is neutralised without
//! discarding deliberate FTS operators such as `OR`.

use std::borrow::Cow;
use std::collections::HashSet;
use std::sync::Arc;

/// Built-in English stopwords excluded from bare-query OR-joins when the
/// operator has not configured `[search.fts] stopwords` at all (issue #953).
/// Deliberately small and boring: high-document-frequency function words
/// that carry no retrieval signal but dominate BM25 through term frequency.
/// This is exactly the list `is_stopword` matched against before the
/// filter became configurable, so an install that never touches
/// `[search.fts]` sees byte-identical behaviour.
pub const DEFAULT_STOPWORDS: &[&str] = &[
    "a", "an", "and", "are", "as", "at", "be", "been", "but", "by", "can", "could", "did", "do",
    "does", "for", "from", "had", "has", "have", "how", "i", "if", "in", "is", "it", "its", "me",
    "my", "of", "on", "or", "our", "she", "should", "so", "that", "the", "their", "them", "they",
    "this", "to", "was", "we", "were", "what", "when", "where", "which", "who", "why", "will",
    "with", "would", "you", "your",
];

/// Normalized stopword set used to filter bare natural-language FTS queries
/// before the OR-join (see [`prepare_fts5_query`]).
///
/// Construction folds every entry to lowercase with [`str::to_lowercase`]
/// (full Unicode case folding, not ASCII-only) so callers never have to
/// pre-normalize a configured list, and comparison at match time folds the
/// query token the same way. This matters specifically for the non-English
/// installs this feature targets: the FTS content index itself is
/// `unicode61 remove_diacritics 2`, so a page body indexes "É"/"é"/"e" as
/// the same posting — an ASCII-only fold would make this filter weaker than
/// the index it feeds, missing a sentence-initial "É" or an all-caps "NÃO"
/// entirely. Folding does NOT strip diacritics, though: "à" and "a" stay
/// distinct tokens here (unlike in the content index), because that would
/// silently change the built-in English default (no accented entries) and
/// because a stopword list should say precisely which spelling to drop.
/// Concretely: what matters for this filter is how a query is actually
/// TYPED, not how wiki content is spelled — content matches through the
/// diacritic-folded index regardless, but a bare query's raw characters are
/// all this filter ever sees. A configured list for an accented language
/// should include both the accented and unaccented forms a user or agent
/// might type (e.g. Portuguese `"e"` and `"é"`, `"nao"` and `"não"`).
///
/// Also note the filter runs on `raw.split_whitespace()` tokens, before any
/// punctuation handling — an entry never matches a token with attached
/// punctuation (`"de,"`, `"que?"`), the same limitation the built-in English
/// list has always had (`"the,"` was never filtered either).
/// The bag-of-words fallback for syntax that does not parse strips quotes
/// and parens first, so `(the` is filtered there, while a word the user put
/// in quotes is always kept.
///
/// Cheap to clone (an `Arc`-shared set) so it can be threaded through a
/// `'static` closure (e.g. `ReaderPool::with_conn`) without re-hashing the
/// list on every search.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FtsStopwords(Arc<HashSet<String>>);

impl FtsStopwords {
    /// Build from an explicit list of words, already validated by the
    /// caller (`ai-memory-cli`'s `Config::load` bounds list size and entry
    /// length for `search.fts.stopwords`, issue #953). Entries are folded
    /// to lowercase here (see the struct doc for the exact fold) so this is
    /// the single place normalization happens.
    #[must_use]
    pub fn new<I, S>(words: I) -> Self
    where
        I: IntoIterator<Item = S>,
        S: AsRef<str>,
    {
        Self(Arc::new(
            words
                .into_iter()
                .map(|w| w.as_ref().to_lowercase())
                .collect(),
        ))
    }

    /// No filtering at all: every token survives the OR-join. This is what
    /// an explicit `stopwords = []` in `config.toml` selects.
    #[must_use]
    pub fn none() -> Self {
        Self(Arc::new(HashSet::new()))
    }

    fn contains(&self, lowercased_token: &str) -> bool {
        self.0.contains(lowercased_token)
    }
}

impl Default for FtsStopwords {
    /// The built-in English list — an absent `[search.fts] stopwords` key
    /// resolves to this, so behaviour is unchanged for every existing
    /// install.
    fn default() -> Self {
        Self::new(DEFAULT_STOPWORDS.iter().copied())
    }
}

/// Sanitize free-text for use in `WHERE pages_fts MATCH ?`.
///
/// Returns an empty string when `raw` is empty/whitespace-only; callers
/// should skip the SQL query in that case.
///
/// Bare multi-word queries are joined with **`OR`**, not the FTS5 default
/// (`AND`). A natural-language query like "cross project search strategy"
/// otherwise requires every word to co-occur in one page — near-zero recall
/// for anything but single keywords. With `OR` + bm25 ranking (callers
/// `ORDER BY rank`), the best-matching pages still surface first. When the
/// caller supplies explicit FTS5 syntax (`OR` / `AND` / `NOT` / `NEAR` /
/// quoted phrases / parens) we preserve it verbatim instead.
///
/// `stopwords` is the operator-configured set (default: [`FtsStopwords::default`],
/// the built-in English list; `FtsStopwords::none()` disables filtering).
///
/// The input is bounded first: a NUL byte is folded to a space, and anything
/// past the 4 KiB cap is dropped, because FTS5's parser is superlinear in the
/// expression it is handed (see `bounded_query_text`).
#[must_use]
pub fn prepare_fts5_query(raw: &str, stopwords: &FtsStopwords) -> String {
    let bounded = bounded_query_text(raw);
    let raw = bounded.as_ref();
    let explicit_syntax = raw.contains('"')
        || raw.contains('(')
        || raw.contains(')')
        || raw
            .split_whitespace()
            .any(|t| matches!(t, "OR" | "AND" | "NOT" | "NEAR"));
    let tokens: Vec<String> = raw
        .split_whitespace()
        // Bare natural-language queries drop stopwords before the
        // OR-join: with no tokenizer-level stopword list, "the/of/a"
        // match nearly every page and BM25 term frequency lets a page
        // with five "the"s outrank the page whose CONTENT matches (seen
        // live: a release-procedure page beaten for a deploy question).
        // Explicit syntax that parses and quoted phrases are untouched, and
        // a query that is ONLY stopwords keeps them all — returning the
        // user's literal terms beats returning nothing. The active list is
        // operator-configured (`[search.fts] stopwords`, issue #953): the
        // built-in English list by default, a custom list for non-English
        // installs, or none at all. Comparison folds Unicode case (matching
        // `FtsStopwords`'s own fold) but never strips diacritics.
        .filter(|t| explicit_syntax || !stopwords.contains(&t.to_lowercase()))
        .flat_map(prepare_fts5_token)
        .collect();
    let tokens = if tokens.is_empty() && !explicit_syntax {
        raw.split_whitespace()
            .flat_map(prepare_fts5_token)
            .collect()
    } else {
        tokens
    };
    if tokens.is_empty() {
        return String::new();
    }
    let separator = if explicit_syntax { " " } else { " OR " };
    let candidate = tokens.join(separator);
    // Natural language trips the explicit-syntax path constantly — `(MoMA)`,
    // `'quoted phrases'`, a sentence that happens to contain OR — and a
    // preserved-verbatim mix can be grammatically invalid FTS5 (a quoted
    // OR-group adjacent to a bare term has no implicit AND, for one). The
    // MATCH must never error on user text, so validate the candidate against
    // a real FTS5 parser and degrade to the always-valid quoted bag of words
    // when it does not parse. Deliberate, well-formed operator queries pass
    // validation and are preserved.
    if fts5_query_parses(&candidate) {
        return candidate;
    }
    // Re-tokenize with every operator neutralised: strip quotes and parens,
    // then quote what remains. The result is a bare OR-join, so it drops
    // stopwords like the bare path above, keeping the words the user put in
    // quotes, and keeping every word when nothing else is left.
    let segments: Vec<&str> = raw.split('"').collect();
    // An unclosed quote leaves an even number of segments; the text after
    // it was never quoted, so it is filtered like the rest.
    let unclosed = segments.len().is_multiple_of(2);
    let last = segments.len() - 1;
    let words: Vec<(&str, bool)> = segments
        .into_iter()
        .enumerate()
        .flat_map(|(index, segment)| {
            let quoted = index % 2 == 1 && !(unclosed && index == last);
            segment
                .split(['(', ')'])
                .flat_map(str::split_whitespace)
                .map(move |word| (word, quoted))
        })
        .collect();
    let content: Vec<&str> = words
        .iter()
        .filter(|(word, quoted)| *quoted || !stopwords.contains(&word.to_lowercase()))
        .map(|(word, _)| *word)
        .collect();
    let kept = if content.is_empty() {
        words.iter().map(|(word, _)| *word).collect()
    } else {
        content
    };
    kept.into_iter()
        .map(quote_fts5_token)
        .collect::<Vec<_>>()
        .join(" OR ")
}

/// Terms stored in the auxiliary CJK index.
///
/// Each maximal Han/Kana/Hangul run contributes its scalar values and its
/// overlapping bigrams. Splitting only those runs keeps punctuation and Latin
/// text out of the derived index, while the bigrams retain enough adjacency
/// to avoid the very broad matches a character-only index would produce.
#[must_use]
pub(crate) fn cjk_index_terms(raw: &str) -> String {
    let mut terms = Vec::new();
    let mut run = Vec::new();
    for ch in raw.chars() {
        if is_cjk(ch) {
            run.push(ch);
        } else {
            append_cjk_terms(&mut terms, &mut run);
        }
    }
    append_cjk_terms(&mut terms, &mut run);
    terms.join(" ")
}

/// A MATCH expression for the CJK portion of a bounded user query.
///
/// `None` is the important common case: callers then execute the existing SQL
/// byte-for-byte. A multi-scalar run becomes an exact phrase of overlapping
/// bigrams; a single scalar uses the unigram fallback.
///
/// `None` also covers a query carrying `AND` / `NOT` / `NEAR`. This stream is
/// auxiliary: callers UNION it into the primary stream, so a run-based
/// OR-join of a `東京 NOT 大阪` query fuses the excluded term's own pages back
/// into the result set — the opposite of what the operator asked for, while
/// the primary stream honours it verbatim. Expanding each operand and
/// rebuilding the expression is the real fix; declining the stream meanwhile
/// restores the pre-#1433 answer instead of an inverted one. `OR` stays,
/// because the expander reproduces it faithfully.
///
/// A leading `title:` / `body:` qualifier is carried onto the run's phrase.
/// This stream is auxiliary — callers UNION it into the primary stream — so a
/// qualifier dropped here searches every column of `pages_cjk_fts`, including
/// `path_search`, and the union widens a query the user deliberately narrowed.
/// The two names are the ones [`prepare_fts5_token`] accepts; any other bare
/// column is left alone, because the primary stream neutralises it too.
#[must_use]
pub(crate) fn prepare_cjk_fts5_query(raw: &str) -> Option<String> {
    let bounded = bounded_query_text(raw);
    if bounded
        .split_whitespace()
        .any(|t| matches!(t, "AND" | "NOT" | "NEAR"))
    {
        return None;
    }
    let mut phrases = Vec::new();
    for token in bounded.split_whitespace() {
        // Grouping must not turn `(title:term)` into an unqualified match.
        // Like prepare_fts5_token, peel opening parentheses before the column:
        // https://www.sqlite.org/fts5.html#fts5_column_filters
        let token = token.trim_start_matches('(');
        let (column, text) = match token.split_once(':') {
            Some((column @ ("title" | "body"), text)) => (Some(column), text),
            _ => (None, token),
        };
        let mut run = Vec::new();
        for ch in text.chars() {
            if is_cjk(ch) {
                run.push(ch);
            } else {
                append_cjk_query(&mut phrases, &mut run, column);
            }
        }
        append_cjk_query(&mut phrases, &mut run, column);
    }
    (!phrases.is_empty()).then(|| phrases.join(" OR "))
}

fn append_cjk_terms(terms: &mut Vec<String>, run: &mut Vec<char>) {
    if run.is_empty() {
        return;
    }
    terms.extend(run.iter().map(char::to_string));
    terms.extend(run.windows(2).map(|pair| pair.iter().collect::<String>()));
    run.clear();
}

fn append_cjk_query(phrases: &mut Vec<String>, run: &mut Vec<char>, column: Option<&str>) {
    if run.is_empty() {
        return;
    }
    let prefix = column.map_or_else(String::new, |column| format!("{column}:"));
    if run.len() == 1 {
        phrases.push(format!("{prefix}\"{}\"", run[0]));
    } else {
        let bigrams = run
            .windows(2)
            .map(|pair| pair.iter().collect::<String>())
            .collect::<Vec<_>>()
            .join(" ");
        phrases.push(format!("{prefix}\"{bigrams}\""));
    }
    run.clear();
}

fn is_cjk(ch: char) -> bool {
    ch.is_alphanumeric()
        && matches!(
        ch as u32,
        0x1100..=0x11ff
            | 0x2e80..=0x2fff
            // CJK Symbols and Punctuation holds a handful of LETTERS among its
            // punctuation: the iteration marks 々 〆 〻 and the kana repeat
            // marks. 々 is how 日々, 時々, 人々, 我々 and 様々 are written, so
            // excluding the block split those words into single scalars on both
            // the index and the query side, leaving the character-only match
            // the bigrams exist to prevent. The `is_alphanumeric` guard above
            // keeps the block's actual punctuation (、 。 「) out.
            | 0x3005..=0x3006
            | 0x3031..=0x3035
            | 0x303b
            | 0x3040..=0x30ff
            | 0x3130..=0x318f
            | 0x31f0..=0x31ff
            | 0x3400..=0x4dbf
            | 0x4e00..=0x9fff
            | 0xa960..=0xa97f
            | 0xac00..=0xd7af
            | 0xd7b0..=0xd7ff
            | 0xf900..=0xfaff
            | 0xff66..=0xff9f
            | 0x20000..=0x323af
        )
}

/// Upper bound (bytes) on the untrusted search text this module tokenises.
///
/// FTS5's query parser is superlinear in the expression it is handed:
/// measured here, 10k distinct terms spent ~60 ms inside
/// [`prepare_fts5_query`], 40k ~1.9 s and 100k ~17.5 s — before the real
/// `MATCH` ran. Search text is a phrase, not a document, so the tail of
/// anything longer is dropped instead of parsed.
const MAX_FTS_QUERY_BYTES: usize = 4_096;

/// Bound untrusted search text before anything parses it.
///
/// A NUL cannot appear in indexed text, but it used to reach the FTS5 parser,
/// which reads a quoted phrase as a C string: `"\0"` fails as an unterminated
/// string, so `memory_query` surfaced `sqlite: unterminated string` as an
/// internal error instead of a result. Folding NUL to a space keeps the rest
/// of the token searchable. Past [`MAX_FTS_QUERY_BYTES`] the text is cut on a
/// UTF-8 boundary; a cut that lands inside a phrase or paren leaves invalid
/// FTS5 syntax, which [`fts5_query_parses`] already degrades to the quoted
/// bag of words. Borrows when neither applies, so the common query path
/// allocates nothing.
fn bounded_query_text(raw: &str) -> Cow<'_, str> {
    let bounded = &raw[..raw.floor_char_boundary(MAX_FTS_QUERY_BYTES)];
    if !bounded.contains('\0') {
        return Cow::Borrowed(bounded);
    }
    Cow::Owned(bounded.replace('\0', " "))
}

/// Whether FTS5's own parser accepts `query`. Uses a throwaway in-memory
/// table (microseconds; search-path frequency, not ingest-path) so the
/// judgement is the engine's, not a re-implementation of its grammar.
fn fts5_query_parses(query: &str) -> bool {
    thread_local! {
        // One probe connection per thread: opening a fresh in-memory
        // database per search query was measurable overhead on the
        // read-pool hot path (post-audit nit).
        static PROBE: Option<rusqlite::Connection> = {
            let conn = rusqlite::Connection::open_in_memory().ok();
            if let Some(c) = &conn
                && c.execute_batch("CREATE VIRTUAL TABLE fts_probe USING fts5(title, body)")
                    .is_err()
            {
                None
            } else {
                conn
            }
        };
    }
    PROBE.with(|probe| {
        probe.as_ref().is_some_and(|conn| {
            conn.query_row(
                "SELECT count(*) FROM fts_probe WHERE fts_probe MATCH ?1",
                [query],
                |row| row.get::<_, i64>(0),
            )
            .is_ok()
        })
    })
}

fn prepare_fts5_token(token: &str) -> Vec<String> {
    // Group parens hug the token (`(title:handoff`, `body:deploy)`), so peel
    // them before the column checks below, which look at the token's start.
    let open = token.len() - token.trim_start_matches('(').len();
    let close = token.len() - token.trim_end_matches(')').len();
    if (open > 0 || close > 0) && open + close < token.len() {
        let mut parts = prepare_fts5_token(&token[open..token.len() - close]);
        if let Some(first) = parts.first_mut() {
            first.insert_str(0, &token[..open]);
        }
        if let Some(last) = parts.last_mut() {
            last.push_str(&token[token.len() - close..]);
        }
        return parts;
    }

    if has_unknown_bare_column(token) {
        return token
            .replace(':', " ")
            .split_whitespace()
            .map(quote_fts5_token)
            .collect();
    }

    // A known column filter keeps its column when its term needs quoting:
    // `title:ai-memory` becomes `title:("ai-memory" OR "ai memory")`, not a
    // quoted phrase that also searches for the literal word `title`.
    // A second colon in the term (`title:10:30`) needs that same quoting:
    // left bare, FTS5 reads the inner `10` as a column name and rejects the
    // whole expression, so the query degrades to the bag of words — which
    // drops the filter and searches every column for the word `title`.
    if !token.contains('"')
        && let Some((column, term)) = token.split_once(':')
        && matches!(column, "title" | "body")
        && !term.is_empty()
        && (should_quote_fts5_token(term) || term.contains(':'))
    {
        return vec![format!("{column}:{}", quote_fts5_token(term))];
    }

    if should_quote_fts5_token(token) {
        vec![quote_fts5_token(token)]
    } else {
        vec![token.to_string()]
    }
}

fn has_unknown_bare_column(token: &str) -> bool {
    token.contains(':')
        && !token.contains('"')
        && !token.starts_with("title:")
        && !token.starts_with("body:")
}

fn should_quote_fts5_token(token: &str) -> bool {
    if token.starts_with('"') && token.ends_with('"') {
        return false;
    }
    // Quote any token carrying ASCII punctuation so FTS5 treats it as a literal
    // phrase instead of erroring on its query grammar — e.g. a filename like
    // `current.md` otherwise yields `fts5: syntax error near "."`. A trailing
    // `*` (the FTS5 prefix operator) is allowed through bare; accented letters
    // and digits are unicode (not ASCII punctuation) so recall keeps accents.
    let core = token.strip_suffix('*').unwrap_or(token);
    // `:` is column syntax (handled by `has_unknown_bare_column`, or preserved
    // for known `title:`/`body:` columns) — it must not trigger quoting here.
    core.chars().any(|c| c.is_ascii_punctuation() && c != ':')
}

fn quote_fts5_token(token: &str) -> String {
    // FTS5 escapes `"` by doubling it. A token carrying a literal quote is an
    // explicit-phrase fragment — keep the simple escaped form (don't expand it).
    if token.contains('"') {
        return format!("\"{}\"", token.replace('"', "\"\""));
    }
    // Otherwise emit BOTH the whole token and a punctuation-stripped sub-token
    // phrase, OR'd, because the content tokenizer and the path index disagree
    // on punctuation:
    //   tokenize = "unicode61 remove_diacritics 2 tokenchars '/_-'"
    // keeps `/ _ -` INSIDE tokens (so a body mention of `ai-memory` indexes as
    // the single token `ai-memory`), while `ops::path_search_text` pre-expands
    // `/ . - _` to spaces in the path index (so a path `ui-refresh-…` indexes
    // the sub-tokens `ui`, `refresh`, …). `.` is a separator either way.
    // Neither form alone matches both: `"ai-memory"` matches the content token
    // but not the split path index; `"ai memory"` matches the path but not the
    // content token. OR-ing the two makes a search for `ai-memory` / `ui-refresh`
    // hit whichever surface indexed it. (Quoting both also neutralises the
    // punctuation that would otherwise be FTS5 query grammar — the original
    // `current.md` → `syntax error` bug.) With no punctuation the two coincide
    // and we emit a single phrase.
    //
    // A trailing `*` is the FTS5 prefix operator. Inside a quoted string it is
    // only a separator, so it moves outside each closing quote instead:
    // `ai-mem*` becomes `("ai-mem"* OR "ai mem"*)`, which FTS5 reads as a
    // prefix match on the phrase's last token.
    let (token, prefix) = match token.strip_suffix('*') {
        Some(core) if core.chars().any(char::is_alphanumeric) => (core, "*"),
        _ => (token, ""),
    };
    let split = token
        .chars()
        .map(|c| if c.is_ascii_punctuation() { ' ' } else { c })
        .collect::<String>();
    let split = split.split_whitespace().collect::<Vec<_>>().join(" ");
    if split.is_empty() || split == token {
        format!("\"{token}\"{prefix}")
    } else {
        format!("(\"{token}\"{prefix} OR \"{split}\"{prefix})")
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Every pre-#953 test called `prepare_fts5_query(raw)` against the
    /// built-in English list; this keeps that call shape so the existing
    /// test bodies below are unchanged (default list reproduces current
    /// behaviour exactly).
    fn prepare_fts5_query(raw: &str) -> String {
        super::prepare_fts5_query(raw, &FtsStopwords::default())
    }

    #[test]
    fn cjk_index_terms_are_bounded_to_cjk_runs() {
        assert_eq!(
            cjk_index_terms("pay 支付服务 now"),
            "支 付 服 务 支付 付服 服务"
        );
        assert_eq!(
            cjk_index_terms("日本語・テスト"),
            "日 本 語 日本 本語 テ ス ト テス スト"
        );
        assert_eq!(cjk_index_terms("plain latin"), "");
        // The iteration mark is a letter in CJK Symbols and Punctuation and
        // must continue a run, or 日々 is indexed as two one-char runs and is
        // unfindable as a unit.
        assert_eq!(
            cjk_index_terms("日々の作業"),
            "日 々 の 作 業 日々 々の の作 作業"
        );
        // Control: the punctuation sharing that block still breaks a run.
        assert_eq!(cjk_index_terms("日、本"), "日 本");
    }

    #[test]
    fn cjk_queries_require_adjacent_bigrams() {
        assert_eq!(
            prepare_cjk_fts5_query("支付服务 status"),
            Some("\"支付 付服 服务\"".to_string())
        );
        assert_eq!(
            prepare_cjk_fts5_query("支 / 韓"),
            Some("\"支\" OR \"韓\"".to_string())
        );
        assert_eq!(prepare_cjk_fts5_query("latin only"), None);
    }

    #[test]
    fn cjk_stream_declines_a_query_the_expander_would_invert() {
        assert_eq!(prepare_cjk_fts5_query("東京 NOT 大阪"), None);
        assert_eq!(prepare_cjk_fts5_query("東京 AND 大阪"), None);
        assert_eq!(prepare_cjk_fts5_query("東京 NEAR 大阪"), None);
        // Control: `OR` is the one operator the run expander reproduces, so
        // the auxiliary stream still runs for it — and a lowercase `not` is
        // not FTS5 syntax, so it must not suppress the stream either.
        let both = Some("\"東京\" OR \"大阪\"".to_string());
        assert_eq!(prepare_cjk_fts5_query("東京 OR 大阪"), both);
        assert_eq!(prepare_cjk_fts5_query("東京 not 大阪"), both);
        assert_parses(&prepare_cjk_fts5_query("東京 OR 大阪").unwrap());
    }

    #[test]
    fn cjk_queries_keep_a_known_column_filter() {
        let title = prepare_cjk_fts5_query("title:支付服务").unwrap();
        assert_eq!(title, "title:\"支付 付服 服务\"");
        assert_parses(&title);
        for query in ["(title:支付服务)", "((title:支付服务))"] {
            let grouped = prepare_cjk_fts5_query(query).unwrap();
            assert_eq!(grouped, title, "query={query}");
            assert_parses(&grouped);
        }
        let body = prepare_cjk_fts5_query("body:支付服务").unwrap();
        assert_eq!(body, "body:\"支付 付服 服务\"");
        assert_parses(&body);
        for query in ["(body:支付服务)", "((body:支付服务))"] {
            let grouped = prepare_cjk_fts5_query(query).unwrap();
            assert_eq!(grouped, body, "query={query}");
            assert_parses(&grouped);
        }
        let alternatives = prepare_cjk_fts5_query("((title:支付服务) OR (body:支付服务))").unwrap();
        assert_eq!(alternatives, format!("{title} OR {body}"));
        assert_parses(&alternatives);
        let single = prepare_cjk_fts5_query("title:支").unwrap();
        assert_eq!(single, "title:\"支\"");
        assert_parses(&single);
        // Control: the bare query is untouched, and a column FTS5 does not
        // know stays unqualified here exactly as the primary stream
        // neutralises it, rather than being emitted as invalid syntax.
        assert_eq!(
            prepare_cjk_fts5_query("支付服务"),
            Some("\"支付 付服 服务\"".to_string())
        );
        let unknown = prepare_cjk_fts5_query("author:支付服务").unwrap();
        assert_eq!(unknown, "\"支付 付服 服务\"");
        assert_parses(&unknown);
    }

    #[test]
    fn cjk_queries_keep_the_japanese_iteration_mark() {
        let query = prepare_cjk_fts5_query("日々").unwrap();
        assert_eq!(query, "\"日々\"");
        assert_parses(&query);
        // Control: the same block's punctuation still separates two runs,
        // so this is a widened letter range and not a widened block.
        assert_eq!(
            prepare_cjk_fts5_query("日、本"),
            Some("\"日\" OR \"本\"".to_string())
        );
    }

    #[test]
    fn cjk_query_input_is_bounded_before_expansion() {
        let query = prepare_cjk_fts5_query(&"日".repeat(10_000)).unwrap();
        assert!(
            query.len() < 12_500,
            "expanded query was {} bytes",
            query.len()
        );
        assert_parses(&query);
    }

    /// The engine, not this module, is the judge of validity: every
    /// prepared query must MATCH without error on a real FTS5 table.
    fn assert_parses(prepared: &str) {
        let conn = rusqlite::Connection::open_in_memory().unwrap();
        conn.execute_batch("CREATE VIRTUAL TABLE t USING fts5(title, body)")
            .unwrap();
        let res = conn.query_row(
            "SELECT count(*) FROM t WHERE t MATCH ?1",
            [prepared],
            |row| row.get::<_, i64>(0),
        );
        assert!(res.is_ok(), "FTS5 rejected {prepared:?}: {res:?}");
    }

    /// Regression (found by the LongMemEval harness): a natural-language
    /// question containing parens and apostrophe-quotes tripped the
    /// explicit-syntax path and produced invalid FTS5 — `fts5: syntax
    /// error near "OR"` — failing the whole memory_query call.
    #[test]
    fn natural_language_with_parens_and_quotes_never_errors() {
        let raw = "How many days passed between my visit to the Museum of \
                   Modern Art (MoMA) and the 'Ancient Civilizations' exhibit \
                   at the Metropolitan Museum of Art?";
        let prepared = prepare_fts5_query(raw);
        assert_parses(&prepared);
        // and the degraded form still carries the searchable words
        assert!(prepared.contains("MoMA"));
        assert!(prepared.contains("Civilizations"));
    }

    /// A NUL byte cannot appear in indexed text, but it used to reach the
    /// FTS5 query parser: a token that starts with one is quoted (`"\0x"`),
    /// and the parser reads the phrase as end-of-input — `sqlite:
    /// unterminated string` came back from `memory_query` as an internal
    /// error. This module's contract is that MATCH never errors on user text.
    #[test]
    fn nul_bytes_never_reach_the_match_expression() {
        for raw in ["\0", "\0x", "x\0", "\0 \0", "(\0)", "a\0b", "title:\0x"] {
            let prepared = prepare_fts5_query(raw);
            assert!(!prepared.contains('\0'), "NUL survived into {prepared:?}");
            if !prepared.is_empty() {
                assert_parses(&prepared);
            }
        }
    }

    /// The FTS5 query parser is superlinear in the number of terms: 100k
    /// distinct terms spent ~17 s inside `prepare_fts5_query` alone, before
    /// the real MATCH ran. Search text is a phrase, not a document, so the
    /// tail of an oversized query must be dropped before the parser sees it.
    #[test]
    fn oversized_query_text_is_bounded_before_parsing() {
        let raw = (0..100_000)
            .map(|i| format!("w{i}"))
            .collect::<Vec<_>>()
            .join(" ");
        let prepared = prepare_fts5_query(&raw);
        assert!(
            prepared.len() <= 32 * 1024,
            "prepared expression is not bounded: {} bytes",
            prepared.len()
        );
        assert_parses(&prepared);
    }

    /// The bound must not touch an ordinary query: this is the exact
    /// expression every search has produced since the OR-join landed.
    #[test]
    fn ordinary_query_text_is_untouched_by_the_bound() {
        assert_eq!(
            prepare_fts5_query("cross project search strategy"),
            "cross OR project OR search OR strategy"
        );
    }

    /// 4 KiB lands mid-character for a 3-byte code point (4096 % 3 != 0); the
    /// cut has to move back to a UTF-8 boundary instead of panicking.
    #[test]
    fn the_bound_cuts_on_a_character_boundary() {
        let raw = "日".repeat(2_000);
        assert!(raw.len() > MAX_FTS_QUERY_BYTES);
        let prepared = prepare_fts5_query(&raw);
        assert_parses(&prepared);
    }

    /// Truncation can land inside a quoted phrase; whatever the cut leaves,
    /// the prepared expression must still parse and MATCH without error.
    #[test]
    fn truncated_explicit_syntax_stays_parseable() {
        let raw = format!("\"unclosed {}", "w ".repeat(3_000));
        let prepared = prepare_fts5_query(&raw);
        assert_parses(&prepared);
    }

    #[test]
    fn hostile_operator_soup_never_errors() {
        for raw in [
            "((",
            "\"unclosed phrase",
            "NEAR(",
            "OR",
            "a OR",
            "OR b",
            "NOT",
            ") misplaced (",
            "col:val:col ( OR \" NEAR",
            "\"\"\"",
        ] {
            let prepared = prepare_fts5_query(raw);
            if !prepared.is_empty() {
                assert_parses(&prepared);
            }
        }
    }

    #[test]
    fn deliberate_well_formed_syntax_is_preserved() {
        let prepared = prepare_fts5_query("title:handoff OR body:deploy");
        assert_parses(&prepared);
        assert_eq!(prepared, "title:handoff OR body:deploy");
        // Quoted phrases were re-tokenized into escaped-quote form long
        // before the validation fallback existed; the contract here is
        // "still valid FTS5 with AND preserved", not byte identity.
        let phrase = prepare_fts5_query("\"exact phrase\" AND deploy");
        assert_parses(&phrase);
        assert!(phrase.contains(" AND deploy"), "{phrase}");
    }

    #[test]
    fn colon_is_not_column_syntax() {
        // Bare multi-word → OR-joined (no explicit operator present).
        // `ai-memory` expands to BOTH the whole token (matches content) and
        // the sub-token phrase (matches the split path index) — see
        // `quote_fts5_token`.
        let q = prepare_fts5_query("pick: handoff ai-memory");
        assert_eq!(q, "\"pick\" OR handoff OR (\"ai-memory\" OR \"ai memory\")");
    }

    #[test]
    fn bare_multi_word_is_or_joined() {
        // The recall fix: every word no longer has to co-occur.
        assert_eq!(
            prepare_fts5_query("cross project search strategy"),
            "cross OR project OR search OR strategy"
        );
    }

    #[test]
    fn portuguese_accented_terms_or_join_and_keep_accents() {
        // PT natural-language query: tokens preserved (accents intact),
        // joined with OR so a page matching any term is found.
        assert_eq!(
            prepare_fts5_query("descrição testes commits"),
            "descrição OR testes OR commits"
        );
    }

    #[test]
    fn single_word_has_no_or() {
        assert_eq!(prepare_fts5_query("handoff"), "handoff");
    }

    /// Regression: a filename like `current.md` used to pass through bare and
    /// FTS5 errored with `syntax error near "."`. Quoting it as a phrase both
    /// avoids the error and matches `architecture-current.md` (the tokens
    /// `current` + `md` are adjacent in the indexed path).
    #[test]
    fn dotted_filename_token_is_quoted() {
        // Whole token OR sub-token phrase. The split form (`current md`)
        // matches the tokenised path; the whole form covers content tokens.
        assert_eq!(
            prepare_fts5_query("current.md"),
            "(\"current.md\" OR \"current md\")"
        );
        assert_eq!(
            prepare_fts5_query("00-index.md"),
            "(\"00-index.md\" OR \"00 index md\")"
        );
        assert_eq!(
            prepare_fts5_query("a/b/c.md"),
            "(\"a/b/c.md\" OR \"a b c md\")"
        );
    }

    /// Regression for the live-found bug: searching `ui-refresh` returned
    /// nothing even though `follow-ups/ui-refresh-scroll-restoration.md`
    /// exists. The old quoting produced `"ui-refresh"`, which FTS5 does NOT
    /// match against the indexed `ui refresh`; the sub-token phrase
    /// `"ui refresh"` does. See the real-FTS5 test in `ops.rs`.
    #[test]
    fn hyphenated_token_quotes_as_subtoken_phrase() {
        assert_eq!(
            prepare_fts5_query("ui-refresh"),
            "(\"ui-refresh\" OR \"ui refresh\")"
        );
        assert_eq!(
            prepare_fts5_query("scroll-restoration"),
            "(\"scroll-restoration\" OR \"scroll restoration\")"
        );
    }

    /// The FTS5 prefix operator (`term*`) must survive — a trailing `*` is not
    /// quoted away.
    #[test]
    fn prefix_star_token_stays_bare() {
        assert_eq!(prepare_fts5_query("curr*"), "curr*");
    }

    /// A punctuated token is quoted, and inside an FTS5 string `*` is only a
    /// separator. The prefix operator has to follow each closing quote, or
    /// `ai-mem*` silently becomes an exact search for `ai-mem`.
    #[test]
    fn prefix_star_survives_quoting_of_punctuated_tokens() {
        assert_eq!(prepare_fts5_query("ai-mem*"), r#"("ai-mem"* OR "ai mem"*)"#);
        assert_eq!(
            prepare_fts5_query("current.m*"),
            r#"("current.m"* OR "current m"*)"#
        );
        // Unknown column syntax is split, and the prefix stays on its term.
        assert_eq!(prepare_fts5_query("pick:hand*"), r#""pick" OR "hand"*"#);
        // A token of punctuation alone has no term to prefix-match.
        assert_eq!(quote_fts5_token("-*"), r#""-*""#);
        for raw in ["ai-mem*", "current.m*", "pick:hand*", "a-* b", "-*", "x.*"] {
            let prepared = prepare_fts5_query(raw);
            if !prepared.is_empty() {
                assert_parses(&prepared);
            }
        }
    }

    #[test]
    fn empty_yields_empty() {
        assert_eq!(prepare_fts5_query("   "), "");
    }

    #[test]
    fn quote_emits_whole_and_subtoken_phrase() {
        // Punctuated identifier → both forms OR'd.
        assert_eq!(
            quote_fts5_token("ai-memory"),
            r#"("ai-memory" OR "ai memory")"#
        );
        // A literal-quote fragment keeps the simple escaped form (no expansion).
        assert_eq!(quote_fts5_token(r#"say "hello""#), r#""say ""hello""""#);
        // No punctuation → single phrase.
        assert_eq!(quote_fts5_token("handoff"), r#""handoff""#);
    }

    #[test]
    fn boolean_operators_are_preserved() {
        assert_eq!(prepare_fts5_query("quick OR slow"), "quick OR slow");
    }

    /// AND is the FTS5 default but operators can be explicit — when the
    /// caller writes one, the OR-join must NOT mangle it into
    /// `foo OR AND OR bar`. Same for NOT and NEAR. (The escape hatch from
    /// the broad-recall default is what makes the OR-join safe to land.)
    #[test]
    fn explicit_and_operator_is_preserved() {
        assert_eq!(prepare_fts5_query("foo AND bar"), "foo AND bar");
    }

    #[test]
    fn explicit_not_operator_is_preserved() {
        assert_eq!(prepare_fts5_query("foo NOT bar"), "foo NOT bar");
    }

    #[test]
    fn explicit_near_operator_is_preserved() {
        assert_eq!(prepare_fts5_query("foo NEAR bar"), "foo NEAR bar");
    }

    /// A query containing a quoted phrase is treated as explicit FTS5
    /// syntax — `"exact phrase" baz` must not become
    /// `"exact" OR "phrase" OR baz` (which destroys the phrase semantics).
    /// The exact assertion is "space-joined, not OR-joined"; what the
    /// individual tokens look like after `prepare_fts5_token` is a
    /// separate concern (and unchanged from pre-#58 behaviour).
    #[test]
    fn quoted_phrase_query_is_not_or_joined() {
        let q = prepare_fts5_query("\"exact phrase\" baz");
        assert!(
            !q.contains(" OR "),
            "explicit quoted-phrase query must not get OR-joined; got {q}"
        );
    }

    /// Same escape-hatch logic for parenthesised sub-expressions —
    /// `(foo OR bar) AND baz` must survive unmangled.
    #[test]
    fn parenthesised_query_is_not_or_joined() {
        let q = prepare_fts5_query("(foo OR bar) AND baz");
        assert!(
            !q.contains("OR (foo"),
            "parens detection must skip OR-join entirely; got {q}"
        );
        assert!(
            q.contains("AND"),
            "explicit AND inside parens query must survive; got {q}"
        );
    }

    #[test]
    fn known_columns_are_preserved() {
        assert_eq!(prepare_fts5_query("title:handoff"), "title:handoff");
    }

    /// A known column filter whose term carries punctuation keeps the filter
    /// on the quoted term, instead of quoting `title:` into the phrase and
    /// searching every column for the word "title".
    #[test]
    fn known_column_filter_survives_quoting_of_its_term() {
        assert_eq!(
            prepare_fts5_query("title:ai-memory"),
            r#"title:("ai-memory" OR "ai memory")"#
        );
        assert_eq!(
            prepare_fts5_query("body:current.md OR title:handoff"),
            r#"body:("current.md" OR "current md") OR title:handoff"#
        );
        for raw in ["title:ai-memory", "body:a.b c", "title:-", "title:a:b"] {
            let prepared = prepare_fts5_query(raw);
            if !prepared.is_empty() {
                assert_parses(&prepared);
            }
        }
    }

    /// A second colon inside a known column's term must not defeat the
    /// filter. FTS5 reads the inner `10` as a column name and rejects the
    /// expression, so the whole query degrades to the quoted bag of words:
    /// that drops the column filter and searches every column for the
    /// literal word "title", the outcome
    /// `known_column_filter_survives_quoting_of_its_term` exists to prevent.
    #[test]
    fn known_column_filter_survives_a_second_colon_in_its_term() {
        assert_eq!(
            prepare_fts5_query("title:10:30"),
            r#"title:("10:30" OR "10 30")"#
        );
        assert_parses(&prepare_fts5_query("title:10:30"));
    }

    /// A column filter inside a group keeps its column: the paren attached
    /// to `title:` must not turn it into an unknown column whose name is
    /// then searched as the literal word "title".
    #[test]
    fn grouped_known_column_filter_keeps_its_column() {
        assert_eq!(prepare_fts5_query("(title:handoff)"), "(title:handoff)");
        assert_eq!(
            prepare_fts5_query("(title:handoff OR body:deploy)"),
            "(title:handoff OR body:deploy)"
        );
        assert_eq!(
            prepare_fts5_query("(title:ai-memory) AND deploy"),
            r#"(title:("ai-memory" OR "ai memory")) AND deploy"#
        );
        for raw in [
            "(title:handoff)",
            "((title:a)",
            "(pick: handoff)",
            "(",
            "(title:)",
        ] {
            let prepared = prepare_fts5_query(raw);
            if !prepared.is_empty() {
                assert_parses(&prepared);
            }
        }
    }

    // --- issue #953: configurable stopword list -----------------------

    /// A custom (Portuguese) list filters the words on that list, but never
    /// touches the English words that are not on it — a Portuguese config
    /// does not accidentally strip English content out of a mixed-language
    /// corpus.
    #[test]
    fn custom_list_filters_configured_words_not_english_ones() {
        let pt = FtsStopwords::new([
            "o", "a", "os", "as", "de", "da", "do", "em", "uma", "um", "com", "para", "que", "se",
            "no", "na",
        ]);
        assert_eq!(
            super::prepare_fts5_query("o teste de integração", &pt),
            "teste OR integração"
        );
        // English stopwords are untouched by the pt list.
        assert_eq!(
            super::prepare_fts5_query("the deploy of the release", &pt),
            "the OR deploy OR of OR the OR release"
        );
    }

    /// `stopwords = []` (`FtsStopwords::none()`) disables filtering
    /// entirely: every bare token, English or not, survives the OR-join.
    #[test]
    fn empty_list_disables_filtering() {
        let none = FtsStopwords::none();
        assert_eq!(
            super::prepare_fts5_query("the of and search", &none),
            "the OR of OR and OR search"
        );
    }

    /// The only-stopwords escape hatch (returning the user's literal terms
    /// beats returning nothing) holds for a custom list too, not just the
    /// built-in English one.
    #[test]
    fn only_configured_stopwords_query_keeps_its_terms() {
        let pt = FtsStopwords::new(["o", "a", "de"]);
        assert_eq!(super::prepare_fts5_query("o a de", &pt), "o OR a OR de");
    }

    /// Explicit FTS5 syntax bypasses the configured filter exactly as it
    /// bypasses the default one.
    #[test]
    fn explicit_syntax_bypasses_custom_list_too() {
        let pt = FtsStopwords::new(["o", "de"]);
        assert_eq!(super::prepare_fts5_query("o AND de", &pt), "o AND de");
    }

    /// Text that trips the explicit-syntax path but does not parse degrades
    /// to the quoted bag of words. That bag is a bare OR-join, so it drops
    /// stopwords the way the bare path does, folding case: `What is the
    /// cause of the stuck deploy` must not OR-in `What`/`is`/`the`/`of` just
    /// because the sentence ends in `(prod)`.
    #[test]
    fn unparseable_syntax_fallback_drops_stopwords() {
        let prepared = prepare_fts5_query("What is the cause of the stuck deploy (prod)");
        assert_parses(&prepared);
        assert_eq!(prepared, "\"cause\" OR \"stuck\" OR \"deploy\" OR \"prod\"");
    }

    /// The fallback filters through the configured list, not the built-in
    /// English one, and `FtsStopwords::none()` keeps every word.
    #[test]
    fn unparseable_syntax_fallback_uses_configured_list() {
        let pt = FtsStopwords::new(["o", "de"]);
        assert_eq!(
            super::prepare_fts5_query("the cause de o deploy (prod)", &pt),
            "\"the\" OR \"cause\" OR \"deploy\" OR \"prod\""
        );
        assert_eq!(
            super::prepare_fts5_query("the cause of the deploy (prod)", &FtsStopwords::none()),
            "\"the\" OR \"cause\" OR \"of\" OR \"the\" OR \"deploy\" OR \"prod\""
        );
    }

    /// A word the user quoted is kept in the fallback even when it is a
    /// stopword, as a quoted phrase is on the explicit-syntax path; the same
    /// word outside quotes is still dropped.
    #[test]
    fn unparseable_syntax_fallback_keeps_quoted_stopwords() {
        let prepared = prepare_fts5_query("\"the\" deploy (prod of the");
        assert_parses(&prepared);
        assert_eq!(prepared, "\"the\" OR \"deploy\" OR \"prod\"");
    }

    /// An unclosed quote does not protect the words after it: they were never
    /// quoted, so their stopwords are dropped like any others.
    #[test]
    fn unparseable_syntax_fallback_filters_after_an_unclosed_quote() {
        let prepared = prepare_fts5_query("the 12\" (rack) of the server");
        assert_parses(&prepared);
        assert_eq!(prepared, "\"12\" OR \"rack\" OR \"server\"");
    }

    /// A fallback query made only of stopwords keeps them all, as the bare
    /// path does: the user's literal words beat an empty search.
    #[test]
    fn unparseable_syntax_fallback_of_only_stopwords_keeps_its_terms() {
        let prepared = prepare_fts5_query("what is (the)");
        assert_parses(&prepared);
        assert_eq!(prepared, "\"what\" OR \"is\" OR \"the\"");
    }

    /// Worked example from the issue / docs: a Portuguese natural-language
    /// question, filtered against a Portuguese stopword list, keeps only the
    /// content words (OR-joined) and preserves their accents — the
    /// stopword comparison is ASCII-lowercase, never diacritic-stripping.
    #[test]
    fn portuguese_stopword_list_worked_example() {
        let pt = FtsStopwords::new([
            "o", "a", "os", "as", "de", "da", "do", "em", "uma", "um", "com", "para", "que", "se",
            "no", "na", "e", "é",
        ]);
        assert_eq!(
            super::prepare_fts5_query("o que fazer quando o teste de integração falha no CI", &pt),
            "fazer OR quando OR teste OR integração OR falha OR CI"
        );
    }

    /// Stopword comparison folds full Unicode case (`str::to_lowercase`, not
    /// ASCII-only) but never strips diacritics. A capitalized or all-caps
    /// accented query token still matches a lowercase configured entry
    /// (sentence-initial "É", all-caps "NÃO"), because the content FTS index
    /// itself is `unicode61 remove_diacritics 2` and this filter would
    /// otherwise be weaker than the index for exactly the languages this
    /// feature targets. But an accent is never folded away: "e" and "é"
    /// stay distinct tokens, so a list must spell out every form a user
    /// might actually type.
    #[test]
    fn stopword_comparison_is_unicode_case_insensitive_but_keeps_diacritics() {
        let pt = FtsStopwords::new(["é", "não"]);
        // Sentence-initial capital + accent still matches the lowercase entry.
        assert_eq!(super::prepare_fts5_query("É teste", &pt), "teste");
        // An all-caps accented entry in the query also folds and matches.
        assert_eq!(super::prepare_fts5_query("NÃO teste", &pt), "teste");
        // "e" (no accent) is a different token entirely and is not filtered
        // by an "é" entry — diacritics are never stripped, only case is folded.
        assert_eq!(super::prepare_fts5_query("e teste", &pt), "e OR teste");
    }
}