Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
fix(ingest): preserve tabular column order during induction
Rudy Celekli6fc3f54cfee9
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3d864085..852b5b62 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -46,6 +46,9 @@ All notable changes to Open Ontologies are documented here.### Fixed+- Preserve XLSX header and Parquet schema order during induction so the first+ filled, unique column remains the identifier candidate, as it is for CSV.+- **The embedding model is loaded once per process rather than once per HTTPsession** (#262). The HTTP arm hands the same server constructor to`StreamableHttpService::new` as a per-session factory, so everything thatdiff --git a/src/ingest.rs b/src/ingest.rsindex 2f6ed7ad..1027fbbb 100644--- a/src/ingest.rs+++ b/src/ingest.rs@@ -419,7 +419,7 @@ impl DataIngester {/// Collect unique keys from all rows, sorted alphabetically./// Column names in the order the FILE gives them, when the format has an- /// order (CSV: the header row), else sorted. `extract_headers` sorts,+ /// order (CSV and XLSX: the header row; Parquet: the schema), else sorted. `extract_headers` sorts,/// which loses which column came first, and the first column is the/// identifier candidate for induction.pub fn headers_in_order(path: &str, rows: &[HashMap<String, String>]) -> Vec<String> {@@ -428,6 +428,31 @@ impl DataIngester {let mut r = csv::ReaderBuilder::new().has_headers(true).from_reader(c.as_bytes());r.headers().ok().map(|h| h.iter().map(|x| x.to_string()).collect())}),+ "xlsx" => (|| -> Result<Vec<String>> {+ use calamine::{open_workbook, Reader, Xlsx};+ let mut workbook: Xlsx<_> = open_workbook(path)?;+ let first_sheet = workbook+ .sheet_names()+ .first()+ .context("XLSX workbook has no sheets")?+ .clone();+ let range = workbook.worksheet_range(&first_sheet)?;+ let header = range.rows().next().context("XLSX file has no rows")?;+ Ok(header.iter().map(Self::calamine_cell_to_string).collect())+ })()+ .ok(),+ "parquet" => (|| -> Result<Vec<String>> {+ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;+ let builder =+ ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path)?)?;+ Ok(builder+ .schema()+ .fields()+ .iter()+ .map(|f| f.name().clone())+ .collect())+ })()+ .ok(),_ => None,};match ordered {diff --git a/tests/fixtures/ingest/header-order.xlsx b/tests/fixtures/ingest/header-order.xlsxnew file mode 100644index 00000000..5c020913Binary files /dev/null and b/tests/fixtures/ingest/header-order.xlsx differdiff --git a/tests/ingest_tabular_order_test.rs b/tests/ingest_tabular_order_test.rsnew file mode 100644index 00000000..2f52e981--- /dev/null+++ b/tests/ingest_tabular_order_test.rs@@ -0,0 +1,69 @@+use std::collections::HashMap;+use std::sync::Arc;++use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray};+use arrow::datatypes::{DataType, Field, Schema};+use open_ontologies::induce::induce;+use open_ontologies::ingest::DataIngester;+use parquet::arrow::ArrowWriter;++fn assert_first_column_is_identifier(path: &str) {+ let rows = DataIngester::parse_file(path).unwrap();+ let headers = DataIngester::headers_in_order(path, &rows);+ assert_eq!(headers, ["record_id", "amount", "date"]);+ let inferred = induce(&rows, &headers, "Orders", "https://example.org/orders/");+ assert_eq!(inferred.id_column, "record_id");+ assert!(!inferred.id_synthesised);+ assert_eq!(inferred.mapping.id_field, "record_id");+}++#[test]+fn xlsx_first_column_remains_the_identifier_candidate() {+ let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))+ .join("tests/fixtures/ingest/header-order.xlsx");+ assert_first_column_is_identifier(path.to_str().unwrap());+}++#[test]+fn parquet_schema_order_remains_the_identifier_candidate() {+ let tmp = tempfile::tempdir().unwrap();+ let path = tmp.path().join("orders.parquet");+ let schema = Arc::new(Schema::new(vec![+ Field::new("record_id", DataType::Utf8, false),+ Field::new("amount", DataType::Int64, false),+ Field::new("date", DataType::Utf8, false),+ ]));+ let columns: Vec<ArrayRef> = vec![+ Arc::new(StringArray::from(vec!["order-b", "order-a"])),+ Arc::new(Int64Array::from(vec![100, 200])),+ Arc::new(StringArray::from(vec!["2026-10-06", "2026-10-07"])),+ ];+ let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();+ let mut writer =+ ArrowWriter::try_new(std::fs::File::create(&path).unwrap(), schema, None).unwrap();+ writer.write(&batch).unwrap();+ writer.close().unwrap();+ assert_first_column_is_identifier(path.to_str().unwrap());+}++#[test]+fn csv_order_and_sorted_fallback_and_extra_keys_are_preserved() {+ let tmp = tempfile::tempdir().unwrap();+ let path = tmp.path().join("orders.csv");+ std::fs::write(+ &path,+ "record_id,amount,date\norder-b,100,2026-10-06\norder-a,200,2026-10-07\n",+ )+ .unwrap();+ assert_first_column_is_identifier(path.to_str().unwrap());+ let mut rows = DataIngester::parse_file(path.to_str().unwrap()).unwrap();+ rows.push(HashMap::from([("extra".into(), "value".into())]));+ assert_eq!(+ DataIngester::headers_in_order(path.to_str().unwrap(), &rows),+ ["record_id", "amount", "date", "extra"]+ );+ assert_eq!(+ DataIngester::headers_in_order("unavailable.json", &rows),+ ["amount", "date", "extra", "record_id"]+ );+}