Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

fix(ingest): preserve tabular column order during induction

Rudy Celekli6fc3f54cfee9
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3d864085..852b5b62 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,9 @@ All notable changes to Open Ontologies are documented here.
### Fixed
+- Preserve XLSX header and Parquet schema order during induction so the first
+ filled, unique column remains the identifier candidate, as it is for CSV.
+
- **The embedding model is loaded once per process rather than once per HTTP
session** (#262). The HTTP arm hands the same server constructor to
`StreamableHttpService::new` as a per-session factory, so everything that
diff --git a/src/ingest.rs b/src/ingest.rs
index 2f6ed7ad..1027fbbb 100644
--- a/src/ingest.rs
+++ b/src/ingest.rs
@@ -419,7 +419,7 @@ impl DataIngester {
/// Collect unique keys from all rows, sorted alphabetically.
/// Column names in the order the FILE gives them, when the format has an
- /// order (CSV: the header row), else sorted. `extract_headers` sorts,
+ /// order (CSV and XLSX: the header row; Parquet: the schema), else sorted. `extract_headers` sorts,
/// which loses which column came first, and the first column is the
/// identifier candidate for induction.
pub fn headers_in_order(path: &str, rows: &[HashMap<String, String>]) -> Vec<String> {
@@ -428,6 +428,31 @@ impl DataIngester {
let mut r = csv::ReaderBuilder::new().has_headers(true).from_reader(c.as_bytes());
r.headers().ok().map(|h| h.iter().map(|x| x.to_string()).collect())
}),
+ "xlsx" => (|| -> Result<Vec<String>> {
+ use calamine::{open_workbook, Reader, Xlsx};
+ let mut workbook: Xlsx<_> = open_workbook(path)?;
+ let first_sheet = workbook
+ .sheet_names()
+ .first()
+ .context("XLSX workbook has no sheets")?
+ .clone();
+ let range = workbook.worksheet_range(&first_sheet)?;
+ let header = range.rows().next().context("XLSX file has no rows")?;
+ Ok(header.iter().map(Self::calamine_cell_to_string).collect())
+ })()
+ .ok(),
+ "parquet" => (|| -> Result<Vec<String>> {
+ use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
+ let builder =
+ ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path)?)?;
+ Ok(builder
+ .schema()
+ .fields()
+ .iter()
+ .map(|f| f.name().clone())
+ .collect())
+ })()
+ .ok(),
_ => None,
};
match ordered {
diff --git a/tests/fixtures/ingest/header-order.xlsx b/tests/fixtures/ingest/header-order.xlsx
new file mode 100644
index 00000000..5c020913
Binary files /dev/null and b/tests/fixtures/ingest/header-order.xlsx differ
diff --git a/tests/ingest_tabular_order_test.rs b/tests/ingest_tabular_order_test.rs
new file mode 100644
index 00000000..2f52e981
--- /dev/null
+++ b/tests/ingest_tabular_order_test.rs
@@ -0,0 +1,69 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int64Array, RecordBatch, StringArray};
+use arrow::datatypes::{DataType, Field, Schema};
+use open_ontologies::induce::induce;
+use open_ontologies::ingest::DataIngester;
+use parquet::arrow::ArrowWriter;
+
+fn assert_first_column_is_identifier(path: &str) {
+ let rows = DataIngester::parse_file(path).unwrap();
+ let headers = DataIngester::headers_in_order(path, &rows);
+ assert_eq!(headers, ["record_id", "amount", "date"]);
+ let inferred = induce(&rows, &headers, "Orders", "https://example.org/orders/");
+ assert_eq!(inferred.id_column, "record_id");
+ assert!(!inferred.id_synthesised);
+ assert_eq!(inferred.mapping.id_field, "record_id");
+}
+
+#[test]
+fn xlsx_first_column_remains_the_identifier_candidate() {
+ let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("tests/fixtures/ingest/header-order.xlsx");
+ assert_first_column_is_identifier(path.to_str().unwrap());
+}
+
+#[test]
+fn parquet_schema_order_remains_the_identifier_candidate() {
+ let tmp = tempfile::tempdir().unwrap();
+ let path = tmp.path().join("orders.parquet");
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("record_id", DataType::Utf8, false),
+ Field::new("amount", DataType::Int64, false),
+ Field::new("date", DataType::Utf8, false),
+ ]));
+ let columns: Vec<ArrayRef> = vec![
+ Arc::new(StringArray::from(vec!["order-b", "order-a"])),
+ Arc::new(Int64Array::from(vec![100, 200])),
+ Arc::new(StringArray::from(vec!["2026-10-06", "2026-10-07"])),
+ ];
+ let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();
+ let mut writer =
+ ArrowWriter::try_new(std::fs::File::create(&path).unwrap(), schema, None).unwrap();
+ writer.write(&batch).unwrap();
+ writer.close().unwrap();
+ assert_first_column_is_identifier(path.to_str().unwrap());
+}
+
+#[test]
+fn csv_order_and_sorted_fallback_and_extra_keys_are_preserved() {
+ let tmp = tempfile::tempdir().unwrap();
+ let path = tmp.path().join("orders.csv");
+ std::fs::write(
+ &path,
+ "record_id,amount,date\norder-b,100,2026-10-06\norder-a,200,2026-10-07\n",
+ )
+ .unwrap();
+ assert_first_column_is_identifier(path.to_str().unwrap());
+ let mut rows = DataIngester::parse_file(path.to_str().unwrap()).unwrap();
+ rows.push(HashMap::from([("extra".into(), "value".into())]));
+ assert_eq!(
+ DataIngester::headers_in_order(path.to_str().unwrap(), &rows),
+ ["record_id", "amount", "date", "extra"]
+ );
+ assert_eq!(
+ DataIngester::headers_in_order("unavailable.json", &rows),
+ ["amount", "date", "extra", "record_id"]
+ );
+}