AI search engine and MCP server for bots and agents. One Rust binary indexes code, docs, logs, JSONL/CSV and PDFs for BM25 full-text, vector and hybrid search, code search without grep, agent memory and RAG. Elasticsearch-compatible API; Corpus Hub serves hundreds of reference memory corpora.
Rust agent-memoryai-agentsbm25claude-codeclicode-searchcursordatabaseelasticsearchfull-text-searchhybrid-searchjsonllocal-firstmcpmcp-servermodel-context-protocolragrustsearch-enginevector-search
autoindex: index INSERT-only seed migrations instead of junking them (Fixes #2877) (#2878)
xerj-org4b38705d5f69
diff --git a/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs b/engine/crates/xerj-autoindex/src/extract/sqlsrc.rsindex 919f8f92c..c583a0c71 100644--- a/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs+++ b/engine/crates/xerj-autoindex/src/extract/sqlsrc.rs@@ -6,7 +6,10 @@//! `CREATE` statements. A file the row parser does not read (Postgres//! `COPY … FROM stdin`, or an `INSERT` the dump lexer skips) falls through//! here from [`super::sqldump`]; schema in that file is still indexed, and a-//! file with no schema is refused with [`DATA_ONLY_DUMP_REASON`].+//! file with no schema is refused with [`DATA_ONLY_DUMP_REASON`] — unless it+//! is a seed migration (#2877): `INSERT`-only and small, or a migration/seed+//! path up to [`BODY_CAP`], with no `COPY … FROM stdin` block. Those are+//! indexed as one SQL code document so their text stays searchable.use super::{ExtractStats, FieldOrigin, RawRecord, Sink};use crate::sniff::{sql_dump_has_rows, SqlLex};@@ -23,6 +26,13 @@ const BODY_CAP: usize = 2 << 20;const STMT_CAP: usize = 64 << 10;const CODE_CHARS: usize = 400;const IMPL_CAP: usize = 16 << 10;+/// #2877: an `INSERT`-only `.sql` file at or under this size is a seed+/// migration (default settings, lookup rows), not a dump, wherever it lives.+const SEED_MAX_BYTES: u64 = 256 << 10;+/// #2877: under a migration / seed path or name, the seed allowance runs up+/// to the stored-body cap. Past it the body would be DDL-only (empty here),+/// so a bigger file is a dump by construction.+const MIGRATION_SEED_MAX_BYTES: u64 = BODY_CAP as u64;/// `scan_file` prints this instead of "no records extracted" when a `.sql`/// file is row data and nothing in it is schema.@@ -45,6 +55,10 @@ enum Phase {struct ScanOut {symbols: Vec<SqlSym>,saw_row_data: bool,+ /// A `COPY … FROM stdin` block was seen: bulk row data, never a seed.+ saw_copy: bool,+ /// Decoded bytes read.+ bytes: u64,/// Full decoded text when the file fits in [`BODY_CAP`], otherwise the/// DDL statements only.body: String,@@ -53,7 +67,10 @@ struct ScanOut {pub fn extract(path: &Path, logical: &Path, gzip: bool, sink: Sink) -> Result<ExtractStats> {let reader = super::open_reader(path, gzip, None)?;let scan = scan_sql(reader)?;- if scan.symbols.is_empty() && scan.saw_row_data {+ if scan.symbols.is_empty()+ && scan.saw_row_data+ && !keeps_data_only_file(logical, scan.bytes, scan.saw_copy)+ {return Ok(ExtractStats::default());}if scan.body.trim().is_empty() && scan.symbols.is_empty() {@@ -62,6 +79,63 @@ pub fn extract(path: &Path, logical: &Path, gzip: bool, sink: Sink) -> Result<ExOk(emit(logical, &scan, sink))}+/// #2877: whether a schema-less `.sql` file of row statements is a seed+/// migration to index as source rather than a data dump to skip.+fn keeps_data_only_file(logical: &Path, bytes: u64, saw_copy: bool) -> bool {+ if saw_copy || bytes > MIGRATION_SEED_MAX_BYTES {+ return false;+ }+ bytes <= SEED_MAX_BYTES || is_migration_path(logical)+}++/// A path that names a migration or seed file: a directory such as+/// `migrations/`, `db/migrate/`, `seeds/`, `seeders/`, `fixtures/`, a file+/// name with `seed`/`migration` in it, a Flyway name (`V3__x.sql`,+/// `R__x.sql`), or a numbered / timestamped name (`0003_x.sql`,+/// `20240101120000_x.sql`).+fn is_migration_path(logical: &Path) -> bool {+ let dir_hit = logical+ .parent()+ .into_iter()+ .flat_map(|p| p.components())+ .filter_map(|c| c.as_os_str().to_str())+ .map(|c| c.to_ascii_lowercase())+ .any(|c| {+ c.contains("migrat")+ || c.contains("seed")+ || c == "fixtures"+ || c == "changelog"+ || c == "changelogs"+ });+ if dir_hit {+ return true;+ }+ let Some(name) = logical.file_name().and_then(|n| n.to_str()) else {+ return false;+ };+ let lower = name.to_ascii_lowercase();+ if lower.contains("seed") || lower.contains("migrat") {+ return true;+ }+ let b = lower.as_bytes();+ // Flyway versioned / repeatable: `V<digits…>__name`, `R__name`.+ if let Some(rest) = lower.strip_prefix('v') {+ let digits = rest+ .bytes()+ .take_while(|c| c.is_ascii_digit() || *c == b'.' || *c == b'_')+ .count();+ if digits > 0 && rest[..digits].bytes().any(|c| c.is_ascii_digit()) && rest.contains("__") {+ return true;+ }+ }+ if lower.starts_with("r__") {+ return true;+ }+ // Numbered / timestamped: at least three leading digits, then `_`/`-`.+ let lead = b.iter().take_while(|c| c.is_ascii_digit()).count();+ lead >= 3 && matches!(b.get(lead), Some(b'_') | Some(b'-'))+}+fn scan_sql(mut reader: impl Read) -> Result<ScanOut> {let mut lex = SqlLex::default();let mut phase = Phase::Sql;@@ -74,6 +148,8 @@ fn scan_sql(mut reader: impl Read) -> Result<ScanOut> {let mut stmt_end_line: usize = 1;let mut symbols = Vec::new();let mut saw_row_data = false;+ let mut saw_copy = false;+ let mut bytes: u64 = 0;let mut ddl = String::new();let mut buf = [0u8; 64 << 10];@@ -82,6 +158,7 @@ fn scan_sql(mut reader: impl Read) -> Result<ScanOut> {if n == 0 {break;}+ bytes += n as u64;for &b in &buf[..n] {if keep_full {if full.len() >= BODY_CAP {@@ -137,6 +214,7 @@ fn scan_sql(mut reader: impl Read) -> Result<ScanOut> {&mut saw_row_data,&mut ddl,) {+ saw_copy = true;phase = Phase::Copy {at_bol: true,idx: 0,@@ -147,15 +225,17 @@ fn scan_sql(mut reader: impl Read) -> Result<ScanOut> {}}}- if !stmt.is_empty() {- let _ = take_statement(+ if !stmt.is_empty()+ && take_statement(&stmt,stmt_line,stmt_end_line,&mut symbols,&mut saw_row_data,&mut ddl,- );+ )+ {+ saw_copy = true;}// A `\.` with no trailing newline is still the end of COPY data.let body = if keep_full {@@ -166,6 +246,8 @@ fn scan_sql(mut reader: impl Read) -> Result<ScanOut> {Ok(ScanOut {symbols,saw_row_data,+ saw_copy,+ bytes,body,})}@@ -822,4 +904,144 @@ mod tests {}}}++ fn direct(path: &Path, logical: &Path) -> (ExtractStats, Vec<crate::extract::RawRecord>) {+ let mut recs = Vec::new();+ let stats = extract(path, logical, false, &mut |r| {+ recs.push(r);+ true+ })+ .unwrap();+ (stats, recs)+ }++ fn inserts(min_bytes: usize) -> String {+ let mut sql = String::from("-- default settings\n");+ let mut i = 0usize;+ while sql.len() <= min_bytes {+ sql.push_str(&format!(+ "INSERT INTO settings (key, value) VALUES ('key_{i}', 'value_{i}');\n"+ ));+ i += 1;+ }+ sql+ }++ fn body_of(recs: &[crate::extract::RawRecord]) -> &str {+ recs.iter()+ .find(|r| r.locator == "code")+ .and_then(|r| r.fields.get("body"))+ .and_then(|v| v.as_str())+ .unwrap_or("")+ }++ /// #2877: a small `INSERT`-only seed migration is source, not a dump.+ /// Fail-before: `extract` returned no records for any schema-less file+ /// with row data.+ #[test]+ fn a_small_insert_only_seed_file_is_indexed_as_one_sql_document() {+ let sql = "INSERT INTO settings (key, value) VALUES ('theme', 'dark');\n\+ INSERT INTO settings (key, value) VALUES ('locale', 'en-GB');\n\+ UPDATE settings SET value = 'on' WHERE key = 'beta';\n";+ let dir = tempfile::tempdir().unwrap();+ let path = write(dir.path(), "settings.sql", sql);+ let (stats, recs) = direct(&path, Path::new("settings.sql"));+ assert_eq!(stats.records, 1, "{recs:?}");+ assert_eq!(recs[0].fields["language"], serde_json::json!("sql"));+ let body = body_of(&recs);+ assert!(body.contains("'dark'") && body.contains("en-GB"), "{body}");+ }++ /// #2877: the size line. Over [`SEED_MAX_BYTES`] an `INSERT`-only file+ /// is indexed only under a migration / seed path, and past+ /// [`MIGRATION_SEED_MAX_BYTES`] it is a dump everywhere.+ #[test]+ fn insert_only_files_past_the_seed_size_need_a_migration_path() {+ let dir = tempfile::tempdir().unwrap();+ let medium = inserts(SEED_MAX_BYTES as usize + 1024);+ let path = write(dir.path(), "medium.sql", &medium);++ let (stats, _) = direct(&path, Path::new("exports/data.sql"));+ assert_eq!(+ stats.records, 0,+ "a medium INSERT file outside migrations is a dump"+ );++ for logical in [+ "db/migrations/0004_seed_settings.sql",+ "db/migrate/20240101120000_defaults.sql",+ "src/main/resources/db/V4__defaults.sql",+ "seeders/defaults.sql",+ ] {+ let (stats, recs) = direct(&path, Path::new(logical));+ assert_eq!(stats.records, 1, "{logical} must be indexed");+ assert!(body_of(&recs).contains("'value_7'"), "{logical}");+ }++ let large = inserts(MIGRATION_SEED_MAX_BYTES as usize + 1024);+ let path = write(dir.path(), "large.sql", &large);+ let (stats, recs) = direct(&path, Path::new("db/migrations/V9__big_seed.sql"));+ assert_eq!(stats.records, 0, "a dump past the body cap stays skipped");+ assert!(recs.is_empty());+ }++ /// #2877: `COPY … FROM stdin` is bulk row data even when tiny and under+ /// a migration path, so it keeps the data-only refusal.+ #[test]+ fn a_copy_block_is_never_a_seed_migration() {+ let sql = "COPY settings (key, value) FROM stdin;\ntheme\tdark\n\\.\n";+ let dir = tempfile::tempdir().unwrap();+ let path = write(dir.path(), "copy.sql", sql);+ let (stats, recs) = direct(&path, Path::new("migrations/V2__seed.sql"));+ assert_eq!(stats.records, 0, "{recs:?}");+ }++ /// #2877 end to end: the field-report shape (comment header, quoted+ /// schema-qualified table, column list) through sniff + dispatch is+ /// searchable and not refused.+ #[test]+ fn an_insert_only_migration_is_searchable_through_autoindex() {+ let sql = "-- Seed default settings\n\+ INSERT INTO \"public\".\"settings\" (\"key\", \"value\") VALUES ('theme', 'dark');\n\+ INSERT INTO \"public\".\"settings\" (\"key\", \"value\") VALUES ('retention_days', '30');\n";+ let dir = tempfile::tempdir().unwrap();+ let path = write(dir.path(), "V3__seed_settings.sql", sql);+ let (stats, recs) = indexed(&path);+ assert!(stats.refusal.is_none(), "{stats:?}");+ assert!(stats.records > 0, "{stats:?}");+ assert!(+ recs.iter().any(|r| r+ .fields+ .values()+ .any(|v| v.as_str().is_some_and(|s| s.contains("retention_days")))),+ "seed text must be indexed: {recs:?}"+ );+ }++ #[test]+ fn migration_paths_and_names() {+ for yes in [+ "migrations/x.sql",+ "db/migrate/x.sql",+ "prisma/migrations/20240101_init/migration.sql",+ "database/seeds/x.sql",+ "V1__init.sql",+ "V1.2__init.sql",+ "R__views.sql",+ "0001_defaults.sql",+ "20240101120000-defaults.sql",+ "seed.sql",+ ] {+ assert!(is_migration_path(Path::new(yes)), "{yes}");+ }+ for no in [+ "dump.sql",+ "backup/full.sql",+ "v1.sql",+ "12_x.sql",+ "vacuum__x.sql",+ ] {+ assert!(!is_migration_path(Path::new(no)), "{no}");+ }+ }}