Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

32.4 KB
Raw Blame
//! One sheet in, one ontology out, with the evidence for every line of it.
//!
//! A spreadsheet is a claim about a domain made by someone who did not write
//! it down: the columns are the properties, the rows are the individuals, the
//! header is the only schema. This module reads that claim back as an OWL
//! class with typed properties, a SHACL shape that the rows themselves
//! satisfy, and a mapping that loads the rows as instances, and it says next
//! to every induced statement WHICH rows made it say so.
//!
//! What it induces is a HYPOTHESIS about the sheet, not a truth about the
//! domain. `sh:maxCount 1` on a column means no row had two values, which is
//! exactly what a one-cell column guarantees and nothing more; `sh:in` on a
//! column means twelve or fewer distinct values were seen, not that a
//! thirteenth is wrong. The output says so, once at the top and once per
//! statement, so a reader can strike the lines the domain does not support.
//!
//! Three choices are deliberate and stated here rather than discovered later:
//!
//! 1. **Cardinality is a shape, not an axiom.** A one-value column could be
//!    rendered `owl:FunctionalProperty`. It is not, because under OWL RL a
//!    functional property with two values does not reject the second row, it
//!    IDENTIFIES the two objects (`prp-fp`). A shape rejects the next row; an
//!    axiom rewrites the world. The next row is what a sheet's author meant.
//! 2. **Observed ranges are evidence, not constraints.** The minimum and
//!    maximum of a numeric column are reported in the shape's description and
//!    never as `sh:minInclusive`, because a price of 0 in a sheet whose
//!    lowest price was 4 is not a violation of anything the author said.
//! 3. **Repeated columns are one property.** `topping1 .. topping7` is one
//!    multi-valued `topping` with `sh:maxCount 7`, which is what the author
//!    meant by numbering them. The group is reported, so a false merge
//!    (`address1`/`address2` as two lines of one address) is visible.

use std::collections::{BTreeMap, BTreeSet, HashMap};

use serde::Serialize;

use crate::mapping::{FieldMapping, MappingConfig};

const XSD: &str = "http://www.w3.org/2001/XMLSchema#";

/// The datatypes this module can tell apart, in the order they are tried.
/// Every filled value of a column must parse for the column to earn the type;
/// one value that does not demotes the column to the next candidate.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum Dt {
    Boolean,
    Integer,
    Decimal,
    Double,
    Date,
    DateTime,
    AnyUri,
    String,
}

impl Dt {
    pub fn iri(self) -> String {
        let local = match self {
            Dt::Boolean => "boolean",
            Dt::Integer => "integer",
            Dt::Decimal => "decimal",
            Dt::Double => "double",
            Dt::Date => "date",
            Dt::DateTime => "dateTime",
            Dt::AnyUri => "anyURI",
            Dt::String => "string",
        };
        format!("{XSD}{local}")
    }

    /// The [`Dt`] an XSD datatype IRI names, for the ones this module knows.
    pub fn from_iri(iri: &str) -> Option<Dt> {
        Some(match iri.strip_prefix(XSD)? {
            "boolean" => Dt::Boolean,
            "integer" => Dt::Integer,
            "decimal" => Dt::Decimal,
            "double" | "float" => Dt::Double,
            "date" => Dt::Date,
            "dateTime" => Dt::DateTime,
            "anyURI" => Dt::AnyUri,
            "string" => Dt::String,
            _ => return None,
        })
    }

    /// Whether `v` is in this datatype's lexical space, as far as this
    /// module's parser goes.
    pub fn accepts(self, v: &str) -> bool {
        let v = v.trim();
        match self {
            Dt::Boolean => matches!(v, "true" | "false" | "TRUE" | "FALSE" | "True" | "False"),
            Dt::Integer => {
                let d = v.strip_prefix(['+', '-']).unwrap_or(v);
                !d.is_empty() && d.chars().all(|c| c.is_ascii_digit())
            }
            Dt::Decimal => {
                let d = v.strip_prefix(['+', '-']).unwrap_or(v);
                let mut dots = 0;
                let mut digits = 0;
                for c in d.chars() {
                    match c {
                        '.' => dots += 1,
                        c if c.is_ascii_digit() => digits += 1,
                        _ => return false,
                    }
                }
                digits > 0 && dots <= 1
            }
            Dt::Double => {
                // A decimal, or a decimal with an exponent (`3e+05`), which a
                // sheet exported from a spreadsheet application will contain
                // for any large round number. INF and NaN as XSD writes them.
                if matches!(v, "INF" | "-INF" | "NaN") {
                    return true;
                }
                let (mant, exp) = match v.split_once(['e', 'E']) {
                    Some((m, e)) => (m, Some(e)),
                    None => (v, None),
                };
                Dt::Decimal.accepts(mant)
                    && exp.is_none_or(|e| {
                        let d = e.strip_prefix(['+', '-']).unwrap_or(e);
                        !d.is_empty() && d.chars().all(|c| c.is_ascii_digit())
                    })
            }
            Dt::Date => {
                let b = v.as_bytes();
                b.len() == 10
                    && b[4] == b'-'
                    && b[7] == b'-'
                    && b.iter().enumerate().all(|(i, c)| i == 4 || i == 7 || c.is_ascii_digit())
            }
            Dt::DateTime => {
                let (d, t) = match v.split_once('T') {
                    Some(x) => x,
                    None => return false,
                };
                if !Dt::Date.accepts(d) {
                    return false;
                }
                let t = t.trim_end_matches('Z');
                let t = t.split(['+']).next().unwrap_or(t);
                let t = if t.get(8..9) == Some("-") { &t[..8] } else { t };
                t.len() >= 8
                    && t.get(2..3) == Some(":")
                    && t.get(5..6) == Some(":")
                    && t.get(..8).is_some_and(|prefix| {
                        prefix.chars().enumerate().all(|(i, c)| i == 2 || i == 5 || c.is_ascii_digit())
                    })
            }
            Dt::AnyUri => (v.starts_with("http://") || v.starts_with("https://")) && !v.contains(' '),
            Dt::String => true,
        }
    }

    /// The narrowest type every value of `values` parses as.
    fn infer<'a>(values: impl Iterator<Item = &'a str> + Clone) -> Dt {
        for dt in [Dt::Boolean, Dt::Integer, Dt::Decimal, Dt::Double, Dt::Date, Dt::DateTime, Dt::AnyUri] {
            if values.clone().all(|v| dt.accepts(v)) {
                return dt;
            }
        }
        Dt::String
    }
}

/// What was measured about one induced property, and what was decided.
#[derive(Debug, Clone, Serialize)]
pub struct Column {
    /// The property's local name.
    pub property: String,
    /// The sheet columns behind it: one, or a numbered group.
    pub columns: Vec<String>,
    pub rows: usize,
    /// Rows with at least one value in these columns.
    pub filled: usize,
    /// Distinct values across the group.
    pub distinct: usize,
    /// The most values one row had across the group.
    pub max_per_row: usize,
    pub datatype: Dt,
    /// `Some(class)`: the values are identifiers of rows of this sheet, so the
    /// property is an object property ranging over the induced class.
    pub refers_to: Option<String>,
    /// Observed enumeration, when the column is a small closed set of strings.
    pub values: Option<Vec<String>>,
    /// Numeric or length bounds actually seen. Evidence, never a constraint.
    pub observed: BTreeMap<&'static str, String>,
    /// One sentence a reader can check against the rows.
    pub evidence: String,
}

#[derive(Debug, Clone, Serialize)]
pub struct Induced {
    pub class: String,
    pub class_iri: String,
    pub base_iri: String,
    /// Original source stem, independent of the normalized class name.
    pub source_stem: String,
    pub id_column: String,
    /// `true` when no column identified the rows and a source-scoped row number was used.
    pub id_synthesised: bool,
    pub rows: usize,
    pub columns: Vec<Column>,
    /// The OWL ontology, Turtle.
    pub ontology_ttl: String,
    /// The SHACL shapes graph, Turtle. The rows satisfy it by construction.
    pub shapes_ttl: String,
    /// The mapping that loads the rows as instances of the class.
    pub mapping: MappingConfig,
    /// What this output is and is not.
    pub means: &'static str,
}

impl Induced {
    /// Serialize the analyzed rows as instances of this induced class.
    ///
    /// When no input column identifies the rows, materialize IDs formed from
    /// the percent-encoded original source stem, a hyphen, and the 1-based
    /// row number in the unused internal ID field. Original cells are retained.
    /// Explicit identifiers use the existing mapping unchanged.
    ///
    /// Replaying the same source in the same row order mints the same IDs.
    /// Sources with the same stem share a namespace; use distinct base IRIs
    /// for unrelated sources with identical stems.
    ///
    /// The exported `MappingConfig` alone cannot synthesize these IDs from raw
    /// input: use this serializer, or supply its generated ID field first.
    pub fn instance_ntriples(&self, rows: &[HashMap<String, String>]) -> String {
        if !self.id_synthesised {
            return self.mapping.rows_to_ntriples(rows);
        }
        let prefix = source_id_prefix(&self.source_stem);
        rows.iter()
            .enumerate()
            .flat_map(|(index, original)| {
                let mut row = original.clone();
                row.insert(self.id_column.clone(), format!("{prefix}-{}", index + 1));
                self.mapping.row_to_triples(&row)
            })
            .collect::<Vec<_>>()
            .join("\n")
    }
}

fn source_id_prefix(stem: &str) -> String {
    const HEX: &[u8; 16] = b"0123456789ABCDEF";
    let mut out = String::new();
    for &byte in stem.as_bytes() {
        if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) {
            out.push(char::from(byte));
        } else {
            out.push('%');
            out.push(char::from(HEX[usize::from(byte >> 4)]));
            out.push(char::from(HEX[usize::from(byte & 15)]));
        }
    }
    out
}

const MEANS: &str = "an induced ontology is a HYPOTHESIS about the sheet, not a truth about the \
                     domain. Every class, property, datatype and shape here was read off the rows \
                     and carries the counts that produced it; the rows satisfy the shapes by \
                     construction, so a clean validation of this sheet against them says nothing. \
                     Cardinality is a shape and not an owl:FunctionalProperty on purpose (an axiom \
                     identifies, a shape rejects), observed numeric ranges are reported and never \
                     constrained, and a numbered column group is one multi-valued property. Strike \
                     what the domain does not support; the evidence column says what each line rests on";

fn local_name(raw: &str) -> String {
    let mut out = String::new();
    let mut upper_next = false;
    for c in raw.trim().chars() {
        if c.is_ascii_alphanumeric() || c == '_' {
            if upper_next && out.is_empty() {
                out.push(c);
            } else if upper_next {
                out.extend(c.to_uppercase());
            } else {
                out.push(c);
            }
            upper_next = false;
        } else {
            upper_next = !out.is_empty();
        }
    }
    if out.is_empty() {
        "column".to_string()
    } else if out.chars().next().is_some_and(|c| c.is_ascii_digit()) {
        format!("c{out}")
    } else {
        out
    }
}

fn class_name(stem: &str) -> String {
    let mut s = String::new();
    let mut up = true;
    for c in stem.chars() {
        if c.is_ascii_alphanumeric() {
            if up {
                s.extend(c.to_uppercase());
            } else {
                s.push(c);
            }
            up = false;
        } else {
            up = true;
        }
    }
    if s.is_empty() {
        "Row".to_string()
    } else if s.chars().next().is_some_and(|c| c.is_ascii_digit()) {
        format!("Sheet{s}")
    } else {
        s
    }
}

/// `topping3` -> `("topping", 3)`, `address 2` -> `("address", 2)`; `abc` -> None.
fn numbered(h: &str) -> Option<(String, u32)> {
    let t = h.trim().trim_end_matches(|c: char| c.is_ascii_digit());
    let digits = &h.trim()[t.len()..];
    if digits.is_empty() || t.is_empty() {
        return None;
    }
    let stem = t.trim_end_matches([' ', '_', '-', '#']);
    if stem.is_empty() {
        return None;
    }
    Some((stem.to_string(), digits.parse().ok()?))
}

fn ttl_str(s: &str) -> String {
    let mut o = String::with_capacity(s.len() + 2);
    o.push('"');
    for c in s.chars() {
        match c {
            '\\' => o.push_str("\\\\"),
            '"' => o.push_str("\\\""),
            '\n' => o.push_str("\\n"),
            '\r' => o.push_str("\\r"),
            '\t' => o.push_str("\\t"),
            c => o.push(c),
        }
    }
    o.push('"');
    o
}

/// Induce class, properties, shapes and mapping from `rows`.
///
/// `headers` fixes the column order (a `HashMap` row has none); `stem` names
/// the class and source namespace (a file stem, usually); `base_iri` roots every minted IRI.
pub fn induce(rows: &[HashMap<String, String>], headers: &[String], stem: &str, base_iri: &str) -> Induced {
    induce_with_source(rows, headers, stem, stem, base_iri)
}

/// Induce a class while retaining an independent source namespace.
///
/// Class overrides affect schema names; source_stem identifies synthetic rows.
/// IDs encode UTF-8 bytes outside ASCII unreserved characters, then append a
/// hyphen and the 1-based row number.
pub fn induce_with_source(
    rows: &[HashMap<String, String>],
    headers: &[String],
    stem: &str,
    source_stem: &str,
    base_iri: &str,
) -> Induced {
    let n = rows.len();
    let class = class_name(stem);
    let ont = format!("{base_iri}ont#");
    let class_iri = format!("{ont}{class}");

    // 1. The identifier: the first column if it is filled and unique in every
    //    row, else any column named like an identifier that is, else a row
    //    number, and the report says which.
    let filled_unique = |h: &String| -> bool {
        let mut seen = BTreeSet::new();
        rows.iter().all(|r| match r.get(h).map(|v| v.trim()) {
            Some(v) if !v.is_empty() => seen.insert(v.to_string()),
            _ => false,
        })
    };
    let id_like = |h: &str| {
        let l = h.to_ascii_lowercase();
        l == "id" || l.ends_with("_id") || l.ends_with("id") || l.contains("key") || l.contains("code")
    };
    let (id_column, id_synthesised) = match headers.first() {
        Some(h) if n > 0 && filled_unique(h) => (h.clone(), false),
        _ => match headers.iter().find(|h| id_like(h) && filled_unique(h)) {
            Some(h) => (h.clone(), false),
            None => {
                let mut field = "__row".to_string();
                while headers.contains(&field) || rows.iter().any(|row| row.contains_key(&field)) {
                    field.push('_');
                }
                (field, true)
            }
        },
    };
    let id_values: BTreeSet<String> = if id_synthesised {
        BTreeSet::new()
    } else {
        rows.iter().filter_map(|r| r.get(&id_column)).map(|v| v.trim().to_string()).collect()
    };

    // 2. Group numbered columns. A stem with two or more members is one
    //    property; a lone `address1` stays a column called address1.
    let mut groups: BTreeMap<String, Vec<(u32, String)>> = BTreeMap::new();
    for h in headers {
        if let Some((stem, k)) = numbered(h) {
            groups.entry(stem).or_default().push((k, h.clone()));
        }
    }
    let mut member_of: HashMap<String, String> = HashMap::new();
    for (stem, members) in &groups {
        if members.len() >= 2 {
            for (_, h) in members {
                member_of.insert(h.clone(), stem.clone());
            }
        }
    }
    let mut units: Vec<(String, Vec<String>)> = Vec::new();
    let mut seen_group: BTreeSet<String> = BTreeSet::new();
    for h in headers {
        if h == &id_column && !id_synthesised {
            continue;
        }
        match member_of.get(h) {
            Some(stem) => {
                if seen_group.insert(stem.clone()) {
                    let mut ms = groups[stem].clone();
                    ms.sort();
                    units.push((stem.clone(), ms.into_iter().map(|(_, h)| h).collect()));
                }
            }
            None => units.push((h.clone(), vec![h.clone()])),
        }
    }

    // 3. Profile each unit.
    let mut columns = Vec::new();
    let mut taken: BTreeSet<String> = BTreeSet::new();
    for (raw_name, cols) in &units {
        let mut property = local_name(raw_name);
        while !taken.insert(property.clone()) {
            property.push('_');
        }
        let per_row: Vec<Vec<&str>> = rows
            .iter()
            .map(|r| {
                cols.iter()
                    .filter_map(|c| r.get(c).map(|v| v.trim()))
                    .filter(|v| !v.is_empty())
                    .collect()
            })
            .collect();
        let filled = per_row.iter().filter(|v| !v.is_empty()).count();
        let max_per_row = per_row.iter().map(|v| v.len()).max().unwrap_or(0);
        let all: Vec<&str> = per_row.iter().flatten().copied().collect();
        let distinct: BTreeSet<&str> = all.iter().copied().collect();
        let datatype = if all.is_empty() { Dt::String } else { Dt::infer(all.iter().copied()) };

        // A column whose every value is an identifier of some row of this
        // sheet refers to the sheet's own class, the only referent a single
        // sheet can offer. Containment alone is not enough: a `quantity`
        // column holding 1..5 is contained in ids 1..100 and refers to
        // nothing, which the first version of this got wrong on a two-row
        // sheet. So the values must be non-numeric, or the column must be
        // named like a reference (`manager_id`, `parent`, `ref`, `key`).
        let named_like_ref = {
            let l = raw_name.to_ascii_lowercase();
            l.ends_with("_id") || l.ends_with("id") && l != "id" || l.contains("ref") || l.contains("parent")
                || l.contains("manager") || l.ends_with("key")
        };
        let refers_to = (!id_synthesised
            && !all.is_empty()
            && !matches!(datatype, Dt::Boolean | Dt::Date | Dt::DateTime)
            && (named_like_ref || !matches!(datatype, Dt::Integer | Dt::Decimal | Dt::Double))
            && all.iter().all(|v| id_values.contains(*v))
            && cols.iter().all(|c| c != &id_column))
        .then(|| class.clone());

        // A closed set needs more than a short column: at least two values
        // (one value is a constant, not an enumeration), at most twelve, and
        // each seen five times over on average. Thirty rows of a four-letter
        // code qualify; thirty rows of names do not.
        let values = (refers_to.is_none()
            && datatype == Dt::String
            && distinct.len() >= 2
            && distinct.len() <= 12
            && all.len() >= 5 * distinct.len())
        .then(|| distinct.iter().map(|v| v.to_string()).collect::<Vec<_>>());

        let mut observed = BTreeMap::new();
        match datatype {
            Dt::Integer | Dt::Decimal | Dt::Double => {
                let nums: Vec<f64> = all.iter().filter_map(|v| v.parse::<f64>().ok()).collect();
                if let (Some(lo), Some(hi)) = (
                    nums.iter().cloned().reduce(f64::min),
                    nums.iter().cloned().reduce(f64::max),
                ) {
                    observed.insert("min", format!("{lo}"));
                    observed.insert("max", format!("{hi}"));
                }
            }
            Dt::String | Dt::AnyUri => {
                if let Some(m) = all.iter().map(|v| v.chars().count()).max() {
                    observed.insert("max_length", m.to_string());
                }
            }
            _ => {}
        }

        let evidence = format!(
            "{filled} of {n} rows filled, {} distinct value(s), at most {max_per_row} per row across {} column(s); every value parses as xsd:{}{}{}",
            distinct.len(),
            cols.len(),
            datatype.iri().rsplit('#').next().unwrap_or(""),
            match &refers_to {
                Some(c) => format!("; every value is the {id_column} of a row{}, so it refers to a {c}", if named_like_ref { " and the column is named like a reference" } else { "" }),
                None => String::new(),
            },
            match &values {
                Some(v) => format!("; {} distinct values over {} occurrences, a closed set as far as this sheet goes", v.len(), all.len()),
                None => String::new(),
            },
        );
        columns.push(Column {
            property,
            columns: cols.clone(),
            rows: n,
            filled,
            distinct: distinct.len(),
            max_per_row,
            datatype,
            refers_to,
            values,
            observed,
            evidence,
        });
    }

    // 4. Render.
    let ctx = Ctx { ont: &ont, class: &class, class_iri: &class_iri, stem: source_stem, n, id_synthesised, id_column: &id_column };
    let ontology_ttl = render_ontology(&ctx, &columns);
    let shapes_ttl = render_shapes(&ctx, &columns);
    let mut mappings = Vec::new();
    for c in &columns {
        for col in &c.columns {
            mappings.push(FieldMapping {
                field: col.clone(),
                predicate: format!("{ont}{}", c.property),
                datatype: c.refers_to.is_none().then(|| c.datatype.iri()),
                class: c.refers_to.as_ref().map(|_| class_iri.clone()),
                lookup: c.refers_to.is_some(),
            });
        }
    }
    let mapping = MappingConfig {
        base_iri: base_iri.to_string(),
        id_field: id_column.clone(),
        class: class_iri.clone(),
        mappings,
    };

    Induced {
        class,
        class_iri,
        base_iri: base_iri.to_string(),
        source_stem: source_stem.to_string(),
        id_column,
        id_synthesised,
        rows: n,
        columns,
        ontology_ttl,
        shapes_ttl,
        mapping,
        means: MEANS,
    }
}

/// What both renderers need to know about the sheet as a whole.
struct Ctx<'a> {
    ont: &'a str,
    class: &'a str,
    class_iri: &'a str,
    stem: &'a str,
    n: usize,
    id_synthesised: bool,
    id_column: &'a str,
}

fn render_ontology(ctx: &Ctx<'_>, columns: &[Column]) -> String {
    let Ctx { ont, class, class_iri, stem, n, id_synthesised, id_column } = *ctx;
    let mut t = String::new();
    t.push_str(&format!("@prefix : <{ont}> .\n"));
    t.push_str("@prefix owl: <http://www.w3.org/2002/07/owl#> .\n");
    t.push_str("@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .\n");
    t.push_str("@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .\n");
    t.push_str("@prefix dcterms: <http://purl.org/dc/terms/> .\n\n");
    t.push_str(&format!(
        "<{}> a owl:Ontology ;\n    rdfs:comment {} .\n\n",
        ont.trim_end_matches('#'),
        ttl_str(&format!(
            "Induced from the sheet {stem:?}: {n} rows, {} properties. {}",
            columns.len(),
            MEANS
        ))
    ));
    t.push_str(&format!(
        ":{class} a owl:Class ;\n    rdfs:label {} ;\n    rdfs:comment {} .\n\n",
        ttl_str(class),
        ttl_str(&format!(
            "One instance per row of {stem:?}, {n} in the sheet. Identified by {}.",
            if id_synthesised {
                "percent-encoded source stem followed by a hyphen and a 1-based row number, because no column was filled and unique in every row".to_string()
            } else {
                format!("the column {id_column:?}, filled and unique in every row")
            }
        ))
    ));
    let _ = class_iri;
    for c in columns {
        let (kind, range) = match &c.refers_to {
            Some(cls) => ("owl:ObjectProperty", format!(":{cls}")),
            None => ("owl:DatatypeProperty", format!("xsd:{}", c.datatype.iri().rsplit('#').next().unwrap_or("string"))),
        };
        t.push_str(&format!(
            ":{p} a {kind} ;\n    rdfs:label {} ;\n    rdfs:domain :{class} ;\n    rdfs:range {range} ;\n    rdfs:comment {} .\n\n",
            ttl_str(&c.columns.join(", ")),
            ttl_str(&c.evidence),
            p = c.property,
        ));
    }
    t
}

fn render_shapes(ctx: &Ctx<'_>, columns: &[Column]) -> String {
    let Ctx { ont, class, class_iri, n, .. } = *ctx;
    let mut t = String::new();
    t.push_str(&format!("@prefix : <{ont}> .\n"));
    t.push_str("@prefix sh: <http://www.w3.org/ns/shacl#> .\n");
    t.push_str("@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .\n\n");
    t.push_str(&format!(
        ":{class}Shape a sh:NodeShape ;\n    sh:targetClass <{class_iri}> ;\n    sh:description {} ;\n",
        ttl_str(&format!(
            "Induced from {n} rows. The rows satisfy this shape by construction; it constrains the NEXT row. {MEANS}"
        ))
    ));
    let mut props = Vec::new();
    for c in columns {
        let mut p = format!("    sh:property [\n        sh:path :{} ;\n", c.property);
        match &c.refers_to {
            Some(_) => p.push_str(&format!("        sh:class <{class_iri}> ;\n        sh:nodeKind sh:IRI ;\n")),
            None => p.push_str(&format!(
                "        sh:datatype xsd:{} ;\n",
                c.datatype.iri().rsplit('#').next().unwrap_or("string")
            )),
        }
        p.push_str(&format!("        sh:maxCount {} ;\n", c.max_per_row.max(1)));
        let mut why = vec![format!(
            "maxCount {} because no row had more values across {} column(s)",
            c.max_per_row.max(1),
            c.columns.len()
        )];
        if c.filled == n && n > 0 {
            p.push_str("        sh:minCount 1 ;\n");
            why.push(format!("minCount 1 because every one of {n} rows was filled"));
        } else {
            why.push(format!("no minCount because {} of {n} rows were empty", n - c.filled));
        }
        if let Some(vals) = &c.values {
            let list: Vec<String> = vals.iter().map(|v| ttl_str(v)).collect();
            p.push_str(&format!("        sh:in ( {} ) ;\n", list.join(" ")));
            why.push(format!(
                "in: {} distinct values over {} filled rows; a value outside this set is new, not wrong",
                vals.len(),
                c.filled
            ));
        }
        for (k, v) in &c.observed {
            why.push(format!("observed {k} {v}, reported and not constrained"));
        }
        p.push_str(&format!("        sh:description {} ;\n    ]", ttl_str(&why.join("; "))));
        props.push(p);
    }
    t.push_str(&props.join(" ;\n"));
    t.push_str(" .\n");
    t
}

#[cfg(test)]
mod tests {
    use super::*;

    fn rows(csv: &str) -> (Vec<HashMap<String, String>>, Vec<String>) {
        let mut lines = csv.trim().lines();
        let headers: Vec<String> = lines.next().unwrap().split(',').map(|s| s.to_string()).collect();
        let rows = lines
            .map(|l| {
                headers
                    .iter()
                    .cloned()
                    .zip(l.split(',').map(|s| s.to_string()))
                    .collect::<HashMap<_, _>>()
            })
            .collect();
        (rows, headers)
    }

    #[test]
    fn datatypes_are_the_narrowest_every_value_parses_as() {
        assert_eq!(Dt::infer(["1", "2", "-3"].into_iter()), Dt::Integer);
        assert_eq!(Dt::infer(["1", "2.5"].into_iter()), Dt::Decimal);
        assert_eq!(Dt::infer(["1", "3e+05"].into_iter()), Dt::Double);
        assert_eq!(Dt::infer(["1", "x"].into_iter()), Dt::String);
        assert_eq!(Dt::infer(["2014-07-11"].into_iter()), Dt::Date);
        assert_eq!(Dt::infer(["2014-07-11T10:00:00Z"].into_iter()), Dt::DateTime);
        assert_eq!(Dt::infer(["true", "false"].into_iter()), Dt::Boolean);
        assert_eq!(Dt::infer(["https://a.b/c"].into_iter()), Dt::AnyUri);
    }

    #[test]
    fn numbered_columns_become_one_property() {
        let (r, h) = rows("name,topping1,topping2,topping3\nA,x,y,\nB,x,,\nC,z,y,w\n");
        let i = induce(&r, &h, "pizza-menu", "http://ex.org/");
        assert_eq!(i.class, "PizzaMenu");
        assert_eq!(i.id_column, "name");
        assert!(!i.id_synthesised);
        assert_eq!(i.columns.len(), 1, "{:?}", i.columns);
        let t = &i.columns[0];
        assert_eq!(t.property, "topping");
        assert_eq!(t.columns, vec!["topping1", "topping2", "topping3"]);
        assert_eq!(t.max_per_row, 3);
        assert_eq!(t.filled, 3);
        assert!(i.shapes_ttl.contains("sh:maxCount 3"), "{}", i.shapes_ttl);
        assert!(i.shapes_ttl.contains("sh:minCount 1"));
        assert!(i.ontology_ttl.contains(":topping a owl:DatatypeProperty"));
        assert!(!i.ontology_ttl.contains("a owl:FunctionalProperty"), "cardinality is a shape, not an axiom");
    }

    #[test]
    fn a_self_reference_is_an_object_property_and_a_small_set_is_an_enumeration() {
        let (r, h) = rows(
            "emp_id,name,manager_id,grade,salary\n1,Ann,,A,100\n2,Bob,1,B,80\n3,Cy,1,B,80.5\n4,Di,2,A,90\n5,Ed,2,B,70\n6,Fi,3,A,60\n7,Gus,3,A,55\n8,Hal,4,B,50\n9,Ida,4,A,45\n10,Jo,5,B,40\n",
        );
        let i = induce(&r, &h, "staff", "http://ex.org/");
        let by = |p: &str| i.columns.iter().find(|c| c.property == p).unwrap_or_else(|| panic!("{p}: {:?}", i.columns));
        assert_eq!(by("manager_id").refers_to.as_deref(), Some("Staff"));
        assert!(i.ontology_ttl.contains(":manager_id a owl:ObjectProperty"));
        assert!(i.shapes_ttl.contains("sh:class <http://ex.org/ont#Staff>"));
        assert_eq!(by("grade").values.as_ref().map(|v| v.len()), Some(2));
        assert!(i.shapes_ttl.contains("sh:in ( \"A\" \"B\" )"));
        assert_eq!(by("salary").datatype, Dt::Decimal);
        assert_eq!(by("salary").observed.get("min").map(String::as_str), Some("40"));
        assert!(!i.shapes_ttl.contains("sh:minInclusive"), "observed ranges are evidence, not constraints");
        assert!(by("manager_id").filled < 10 && !i.shapes_ttl.contains("sh:path :manager_id ;\n        sh:class <http://ex.org/ont#Staff> ;\n        sh:nodeKind sh:IRI ;\n        sh:maxCount 1 ;\n        sh:minCount 1"));
        // name is not an enumeration: ten distinct over ten values.
        assert!(by("name").values.is_none());
    }

    /// The false positive the first version had: small integers contained in
    /// the id range are a quantity, not a reference.
    #[test]
    fn an_integer_column_inside_the_id_range_is_not_a_reference() {
        let (r, h) = rows("id,label,n\n1,one,1\n2,two,2\n3,three,1\n");
        let i = induce(&r, &h, "nums", "http://ex.org/");
        let n = i.columns.iter().find(|c| c.property == "n").unwrap();
        assert!(n.refers_to.is_none(), "{:?}", n);
        assert_eq!(n.datatype, Dt::Integer);
        // ...but the same values under a reference-like name are one.
        let (r, h) = rows("id,label,parent_id\n1,one,1\n2,two,2\n3,three,1\n");
        let i = induce(&r, &h, "nums", "http://ex.org/");
        assert_eq!(i.columns.iter().find(|c| c.property == "parent_id").unwrap().refers_to.as_deref(), Some("Nums"));
    }

    #[test]
    fn no_identifier_means_a_row_number_and_says_so() {
        let (r, h) = rows("colour,size\nred,1\nred,2\n");
        let i = induce(&r, &h, "swatches", "http://ex.org/");
        assert!(i.id_synthesised);
        assert_eq!(i.id_column, "__row");
        assert!(i.ontology_ttl.contains("row number, because no column was filled and unique"));
        assert_eq!(i.columns.len(), 2);
    }

    #[test]
    fn the_turtle_parses_and_the_mapping_loads_rows_as_instances() {
        let (r, h) = rows("id,label,n\n1,one,1\n2,two,2\n");
        let i = induce(&r, &h, "nums", "http://ex.org/");
        let g = crate::graph::GraphStore::new();
        g.load_turtle(&i.ontology_ttl, None).expect("ontology turtle parses");
        g.load_turtle(&i.shapes_ttl, None).expect("shapes turtle parses");
        let nt = i.mapping.rows_to_ntriples(&r);
        assert!(nt.contains("<http://ex.org/1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://ex.org/ont#Nums>"), "{nt}");
        assert!(nt.contains("\"1\"^^<http://www.w3.org/2001/XMLSchema#integer>"), "{nt}");
    }
}