Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.
Rust ai-nativedescription-logicsformal-verificationinfrastructure-as-codeisabelle-languageknowledge-graphleanlean4mcpmcp-serverontologyowlowl2reasoningrustshaclsparqltheorem-proving
use anyhow::{Context, Result};
use std::collections::HashMap;
use std::fs;
/// Data ingester that parses structured data files into rows of key-value pairs.
pub struct DataIngester;
/// Convert a serde_json::Value to a flat string representation.
fn json_value_to_string(v: &serde_json::Value) -> String {
match v {
serde_json::Value::String(s) => s.clone(),
serde_json::Value::Number(n) => n.to_string(),
serde_json::Value::Bool(b) => b.to_string(),
serde_json::Value::Null => String::new(),
other => other.to_string(),
}
}
impl DataIngester {
/// Detect the file format from the file extension.
/// Returns one of: "csv", "json", "ndjson", "xml", "yaml", "xlsx", "parquet".
/// Defaults to "csv" if the extension is unrecognised.
pub fn detect_format(path: &str) -> &'static str {
let ext = std::path::Path::new(path)
.extension()
.and_then(|e| e.to_str())
.unwrap_or("")
.to_lowercase();
match ext.as_str() {
"csv" => "csv",
"json" => "json",
"jsonl" | "ndjson" => "ndjson",
"xml" => "xml",
"yaml" | "yml" => "yaml",
"xlsx" => "xlsx",
"parquet" => "parquet",
_ => "csv",
}
}
/// Parse CSV content with headers into rows.
pub fn parse_csv(content: &str) -> Result<Vec<HashMap<String, String>>> {
let mut reader = csv::Reader::from_reader(content.as_bytes());
let headers: Vec<String> = reader
.headers()
.context("Failed to read CSV headers")?
.iter()
.map(|h| h.to_string())
.collect();
let mut rows = Vec::new();
for result in reader.records() {
let record = result.context("Failed to read CSV record")?;
let mut row = HashMap::new();
for (i, value) in record.iter().enumerate() {
if let Some(key) = headers.get(i) {
row.insert(key.clone(), value.to_string());
}
}
rows.push(row);
}
Ok(rows)
}
/// Parse a JSON string. Accepts a JSON array of objects or a single object.
pub fn parse_json(content: &str) -> Result<Vec<HashMap<String, String>>> {
let value: serde_json::Value =
serde_json::from_str(content).context("Failed to parse JSON")?;
match value {
serde_json::Value::Array(arr) => {
let mut rows = Vec::new();
for item in &arr {
if let serde_json::Value::Object(map) = item {
let row: HashMap<String, String> = map
.iter()
.map(|(k, v)| (k.clone(), json_value_to_string(v)))
.collect();
rows.push(row);
}
}
Ok(rows)
}
serde_json::Value::Object(map) => {
let row: HashMap<String, String> = map
.iter()
.map(|(k, v)| (k.clone(), json_value_to_string(v)))
.collect();
Ok(vec![row])
}
_ => Ok(Vec::new()),
}
}
/// Parse newline-delimited JSON (NDJSON / JSON Lines).
pub fn parse_ndjson(content: &str) -> Result<Vec<HashMap<String, String>>> {
let mut rows = Vec::new();
for line in content.lines() {
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
let value: serde_json::Value =
serde_json::from_str(trimmed).context("Failed to parse NDJSON line")?;
if let serde_json::Value::Object(map) = value {
let row: HashMap<String, String> = map
.iter()
.map(|(k, v)| (k.clone(), json_value_to_string(v)))
.collect();
rows.push(row);
}
}
Ok(rows)
}
/// Parse a YAML string containing an array of objects.
pub fn parse_yaml(content: &str) -> Result<Vec<HashMap<String, String>>> {
let value: serde_yaml::Value =
serde_yaml::from_str(content).context("Failed to parse YAML")?;
match value {
serde_yaml::Value::Sequence(seq) => {
let mut rows = Vec::new();
for item in &seq {
if let serde_yaml::Value::Mapping(map) = item {
let mut row = HashMap::new();
for (k, v) in map {
let key = match k {
serde_yaml::Value::String(s) => s.clone(),
other => format!("{other:?}"),
};
let val = match v {
serde_yaml::Value::String(s) => s.clone(),
serde_yaml::Value::Number(n) => n.to_string(),
serde_yaml::Value::Bool(b) => b.to_string(),
serde_yaml::Value::Null => String::new(),
other => format!("{other:?}"),
};
row.insert(key, val);
}
rows.push(row);
}
}
Ok(rows)
}
_ => Ok(Vec::new()),
}
}
/// Parse XML with a `<root><record>...</record></root>` structure.
/// Depth 1 = root element, depth 2 = record boundary, depth 3 = field elements.
pub fn parse_xml(content: &str) -> Result<Vec<HashMap<String, String>>> {
use quick_xml::events::Event;
use quick_xml::reader::Reader;
let mut reader = Reader::from_str(content);
// Trimming every event destroys spaces when a field is split by a
// comment or CDATA boundary. Normalize only the completed field.
reader.config_mut().trim_text(false);
let mut rows: Vec<HashMap<String, String>> = Vec::new();
let mut current_row: Option<HashMap<String, String>> = None;
let mut current_field: Option<String> = None;
// Keep raw text until the field closes: XML whitespace is trimmed
// before entity decoding, while CDATA is literal and never unescaped.
let mut field_parts: Vec<(String, bool)> = Vec::new();
let mut depth: u32 = 0;
loop {
match reader.read_event() {
Ok(Event::Start(e)) => {
depth += 1;
match depth {
1 => {
// Root element — nothing to do
}
2 => {
// Record start
current_row = Some(HashMap::new());
}
3 => {
// Field element start
let name = String::from_utf8_lossy(e.name().as_ref()).to_string();
field_parts.clear();
current_field = Some(name);
}
_ => {}
}
}
Ok(Event::Text(e)) => {
if depth == 3 && current_row.is_some() && current_field.is_some() {
let text = std::str::from_utf8(e.as_ref())
.context("Failed to decode XML field text")?;
field_parts.push((text.to_string(), true));
}
}
Ok(Event::CData(e)) => {
if depth == 3 && current_row.is_some() && current_field.is_some() {
let text = e.decode().context("Failed to decode XML field CDATA")?;
field_parts.push((text.into_owned(), false));
}
}
Ok(Event::End(_)) => {
match depth {
2 => {
// Record end — push row
if let Some(row) = current_row.take() {
rows.push(row);
}
}
3 => {
// Keep quick-xml's existing XML-whitespace policy,
// including significant entity-encoded spaces.
for (raw, _) in &mut field_parts {
*raw = raw.trim_start_matches([' ', '\t', '\r', '\n']).to_string();
if !raw.is_empty() {
break;
}
}
for (raw, _) in field_parts.iter_mut().rev() {
*raw = raw.trim_end_matches([' ', '\t', '\r', '\n']).to_string();
if !raw.is_empty() {
break;
}
}
let mut value = String::new();
for (raw, escaped) in &field_parts {
if *escaped {
value.push_str(
&quick_xml::escape::unescape(raw)
.context("Failed to decode XML field text")?,
);
} else {
value.push_str(raw);
}
}
// An empty field did not create or overwrite a key.
if !value.is_empty()
&& let (Some(row), Some(field)) = (&mut current_row, ¤t_field)
{
row.insert(field.clone(), value);
}
field_parts.clear();
current_field = None;
}
_ => {}
}
depth -= 1;
}
Ok(Event::Eof) => {
// quick-xml accepts EOF with unclosed elements. Do not
// report a completed prefix as a successful ingestion.
if depth != 0 {
anyhow::bail!(
"XML parse error: unexpected EOF with {depth} unclosed elements"
);
}
break;
}
Err(e) => return Err(anyhow::anyhow!("XML parse error: {e}")),
_ => {}
}
}
Ok(rows)
}
/// Parse an Excel (.xlsx) file. First row is treated as headers.
pub fn parse_xlsx_file(path: &str) -> Result<Vec<HashMap<String, String>>> {
use calamine::{open_workbook, Reader, Xlsx};
let mut workbook: Xlsx<_> =
open_workbook(path).context("Failed to open XLSX workbook")?;
let sheet_names = workbook.sheet_names().to_vec();
let first_sheet = sheet_names
.first()
.context("XLSX workbook has no sheets")?
.clone();
let range = workbook
.worksheet_range(&first_sheet)
.context("Failed to read XLSX worksheet")?;
let mut row_iter = range.rows();
let header_row = row_iter.next().context("XLSX file has no rows")?;
let headers: Vec<String> = header_row
.iter()
.map(Self::calamine_cell_to_string)
.collect();
let mut rows = Vec::new();
for data_row in row_iter {
let mut row = HashMap::new();
for (i, cell) in data_row.iter().enumerate() {
if let Some(key) = headers.get(i) {
row.insert(key.clone(), Self::calamine_cell_to_string(cell));
}
}
rows.push(row);
}
Ok(rows)
}
/// Convert a calamine Data cell to a string.
fn calamine_cell_to_string(cell: &calamine::Data) -> String {
match cell {
calamine::Data::Int(i) => i.to_string(),
calamine::Data::Float(f) => f.to_string(),
calamine::Data::String(s) => s.clone(),
calamine::Data::Bool(b) => b.to_string(),
calamine::Data::DateTime(dt) if dt.is_datetime() => dt
.as_datetime()
.map(|value| value.format("%Y-%m-%dT%H:%M:%S%.f").to_string())
.unwrap_or_else(|| dt.to_string()),
calamine::Data::DateTime(dt) => dt.to_string(),
calamine::Data::DateTimeIso(s) => s.clone(),
calamine::Data::DurationIso(s) => s.clone(),
calamine::Data::Error(e) => format!("{e:?}"),
calamine::Data::Empty => String::new(),
}
}
/// Parse a Parquet file into rows.
pub fn parse_parquet_file(path: &str) -> Result<Vec<HashMap<String, String>>> {
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use std::fs::File;
let file = File::open(path).context("Failed to open Parquet file")?;
let builder = ParquetRecordBatchReaderBuilder::try_new(file)
.context("Failed to create Parquet reader builder")?;
let reader = builder.build().context("Failed to build Parquet reader")?;
let mut rows = Vec::new();
for batch_result in reader {
let batch: arrow::array::RecordBatch =
batch_result.context("Failed to read Parquet record batch")?;
let schema = batch.schema();
let num_rows = batch.num_rows();
let num_cols = batch.num_columns();
for row_idx in 0..num_rows {
let mut row = HashMap::new();
for col_idx in 0..num_cols {
let field = schema.field(col_idx);
let col = batch.column(col_idx);
let value = if col.is_null(row_idx) {
String::new()
} else {
Self::arrow_array_value_to_string(col.as_ref(), row_idx, field.data_type())
.with_context(|| {
format!(
"Failed to format Parquet column {} row {row_idx}",
field.name()
)
})?
};
row.insert(field.name().clone(), value);
}
rows.push(row);
}
}
Ok(rows)
}
/// Convert an Arrow array value at a given index to a string.
fn arrow_array_value_to_string(
array: &dyn arrow::array::Array,
idx: usize,
data_type: &arrow::datatypes::DataType,
) -> Result<String> {
use arrow::array::*;
use arrow::datatypes::DataType as ArrowType;
Ok(match data_type {
ArrowType::Boolean => {
let a = array.as_any().downcast_ref::<BooleanArray>().unwrap();
a.value(idx).to_string()
}
ArrowType::Int8 => {
let a = array.as_any().downcast_ref::<Int8Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::Int16 => {
let a = array.as_any().downcast_ref::<Int16Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::Int32 => {
let a = array.as_any().downcast_ref::<Int32Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::Int64 => {
let a = array.as_any().downcast_ref::<Int64Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::UInt8 => {
let a = array.as_any().downcast_ref::<UInt8Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::UInt16 => {
let a = array.as_any().downcast_ref::<UInt16Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::UInt32 => {
let a = array.as_any().downcast_ref::<UInt32Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::UInt64 => {
let a = array.as_any().downcast_ref::<UInt64Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::Float32 => {
let a = array.as_any().downcast_ref::<Float32Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::Float64 => {
let a = array.as_any().downcast_ref::<Float64Array>().unwrap();
a.value(idx).to_string()
}
ArrowType::Utf8 => {
let a = array.as_any().downcast_ref::<StringArray>().unwrap();
a.value(idx).to_string()
}
ArrowType::LargeUtf8 => {
let a = array.as_any().downcast_ref::<LargeStringArray>().unwrap();
a.value(idx).to_string()
}
// Format this cell, never the whole column's Debug view. Dates,
// timestamps and scaled decimals otherwise include every other
// row's value in the string assigned to this one.
_ => arrow::util::display::array_value_to_string(array, idx)?,
})
}
/// Read a whole text file into memory, refusing anything over the ingest cap.
/// Ingest reads the entire file at once, so an unbounded read is an
/// out-of-memory lever for any caller that can point the tool at a large file.
fn read_to_string_capped(path: &str) -> Result<String> {
const MAX_INGEST_BYTES: u64 = 512 * 1024 * 1024;
let meta =
fs::metadata(path).with_context(|| format!("Failed to stat file: {path}"))?;
if meta.len() > MAX_INGEST_BYTES {
anyhow::bail!(
"file {path} is {} bytes, over the {MAX_INGEST_BYTES}-byte ingest cap",
meta.len()
);
}
fs::read_to_string(path).with_context(|| format!("Failed to read file: {path}"))
}
/// Dispatch to the correct parser based on detected format.
/// For text formats, reads the file content first.
pub fn parse_file(path: &str) -> Result<Vec<HashMap<String, String>>> {
Self::parse_file_with_format(path, None)
}
/// Parse the explicitly requested format, or infer it from the extension.
pub fn parse_file_with_format(
path: &str,
format: Option<&str>,
) -> Result<Vec<HashMap<String, String>>> {
let format = format.unwrap_or_else(|| Self::detect_format(path));
match format {
"csv" => {
let content = Self::read_to_string_capped(path)?;
Self::parse_csv(&content)
}
"json" => {
let content = Self::read_to_string_capped(path)?;
Self::parse_json(&content)
}
"ndjson" => {
let content = Self::read_to_string_capped(path)?;
Self::parse_ndjson(&content)
}
"yaml" => {
let content = Self::read_to_string_capped(path)?;
Self::parse_yaml(&content)
}
"xml" => {
let content = Self::read_to_string_capped(path)?;
Self::parse_xml(&content)
}
"xlsx" => Self::parse_xlsx_file(path),
"parquet" => Self::parse_parquet_file(path),
_ => anyhow::bail!("Unsupported data format: {format}"),
}
}
/// Collect unique keys from all rows, sorted alphabetically.
/// Column names in the order the FILE gives them, when the format has an
/// order (CSV and XLSX: the header row; Parquet: the schema), else sorted. `extract_headers` sorts,
/// which loses which column came first, and the first column is the
/// identifier candidate for induction.
pub fn headers_in_order(path: &str, rows: &[HashMap<String, String>]) -> Vec<String> {
let ordered: Option<Vec<String>> = match Self::detect_format(path) {
"csv" => std::fs::read_to_string(path).ok().and_then(|c| {
let mut r = csv::ReaderBuilder::new().has_headers(true).from_reader(c.as_bytes());
r.headers().ok().map(|h| h.iter().map(|x| x.to_string()).collect())
}),
"xlsx" => (|| -> Result<Vec<String>> {
use calamine::{open_workbook, Reader, Xlsx};
let mut workbook: Xlsx<_> = open_workbook(path)?;
let first_sheet = workbook
.sheet_names()
.first()
.context("XLSX workbook has no sheets")?
.clone();
let range = workbook.worksheet_range(&first_sheet)?;
let header = range.rows().next().context("XLSX file has no rows")?;
Ok(header.iter().map(Self::calamine_cell_to_string).collect())
})()
.ok(),
"parquet" => (|| -> Result<Vec<String>> {
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
let builder =
ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(path)?)?;
Ok(builder
.schema()
.fields()
.iter()
.map(|f| f.name().clone())
.collect())
})()
.ok(),
_ => None,
};
match ordered {
Some(h) if !h.is_empty() => {
// The file's order for the columns it names, then any key a
// row carries that the header did not.
let mut out = h;
for k in Self::extract_headers(rows) {
if !out.contains(&k) {
out.push(k);
}
}
out
}
_ => Self::extract_headers(rows),
}
}
pub fn extract_headers(rows: &[HashMap<String, String>]) -> Vec<String> {
let mut keys: Vec<String> = rows
.iter()
.flat_map(|row| row.keys().cloned())
.collect::<std::collections::HashSet<String>>()
.into_iter()
.collect();
keys.sort();
keys
}
}