Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

5.4 KB
Raw Blame
"""Normalise document access control into one queryable shape.

Why this exists: independent per-document derivation produced three different
models of the same notion. Four documents carry a reified chain
(:Document -> :HasDocumentControl -> :DocumentControl -> :HasAccessControlGroup
-> :AclGroup -> :AclGroupName "..."), some carry a flat :aclGroup literal, and
the rest lost the field entirely when the extractor split the header. None of
that is queryable as access control, because a filter cannot compare a group
name that lives behind an opaque entity node in one document and a literal in
another.

The document headers are the authoritative source: every header states both a
Classification and an Acl Group. This module reads them directly and asserts
one flat, uniform shape on every :DOC_* individual, so retrieval can filter by
group with a single triple pattern rather than a three-hop path that only
matches part of the corpus.

Output is additive. Nothing already in the store is removed, so the reified
chain remains for anyone who wants to see what the extractor actually did.
"""

from __future__ import annotations

import pathlib
import re
import zipfile

CORPUS = pathlib.Path("demo/corpus/dcat-us")
OUT = pathlib.Path("demo/derived/_acl.ttl")

PREFIX = "https://w3id.org/dcat-us-demo#"

# The header is a flat run of "Field Value" pairs once the XML is stripped, so
# each field terminates at the next known field name rather than at whitespace.
FIELD_END = r"(?: SYNTHETIC| Title| Owner| Classification| Acl Group| Version| Doc Id|$)"


def doc_text(path: pathlib.Path) -> str:
    xml = zipfile.ZipFile(path).read("word/document.xml").decode("utf8", "ignore")
    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", xml))


def field(text: str, name: str) -> str | None:
    m = re.search(rf"{name} (.+?){FIELD_END}", text)
    return m.group(1).strip() if m else None


def safe_iri(doc_id: str) -> str:
    """Match the pipeline's IRI convention: DOC-202 -> DOC_DOC_202."""
    return "DOC_" + re.sub(r"[^A-Za-z0-9]", "_", doc_id)


def extract() -> list[dict[str, str]]:
    rows: list[dict[str, str]] = []
    for path in sorted(CORPUS.glob("*.docx")):
        text = doc_text(path)
        doc_id = field(text, "Doc Id")
        if not doc_id:
            continue
        groups = field(text, "Acl Group")
        rows.append(
            {
                "iri": safe_iri(doc_id),
                "doc_id": doc_id,
                "classification": field(text, "Classification") or "Unclassified",
                # Stored one triple per group: a comma-joined literal cannot be
                # filtered by membership without string matching, which fails
                # the moment a group name is a substring of another.
                "groups": [g.strip() for g in (groups or "").split(",") if g.strip()],
            }
        )
    return rows


def render(rows: list[dict[str, str]]) -> str:
    out = [f"@prefix : <{PREFIX}> .", "@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .", ""]
    out.append("# Access control normalised from the document headers, which are")
    out.append("# authoritative and cover the whole corpus. Generated by demo/acl_normalise.py.")
    out.append("")
    for r in rows:
        groups = " , ".join(f'"{g}"' for g in r["groups"]) or '"unassigned"'
        out.append(f':{r["iri"]} :aclGroup {groups} ;')
        out.append(f'    :classification "{r["classification"]}" ;')
        out.append(f'    :docId "{r["doc_id"]}" .')
        out.append("")
    return "\n".join(out)


MARKER = "# --- normalised access control (demo/acl_normalise.py) ---"
STORE = pathlib.Path("demo/derived/_store.ttl")


def merge_into_store(rows: list[dict[str, str]]) -> str:
    """Append the ACL block to the store the app actually loads.

    Idempotent: a previous block is replaced rather than duplicated, so this
    can run after every build without the store growing a new copy each time.
    The block is appended rather than woven in because the store is generated,
    and keeping the addition in one contiguous, marked region makes it obvious
    on inspection which triples came from here and which the pipeline derived.
    """
    if not STORE.exists():
        return f"store not found at {STORE}, skipped merge"

    text = STORE.read_text()
    if MARKER in text:
        text = text[: text.index(MARKER)].rstrip() + "\n"

    body = []
    for r in rows:
        groups = " , ".join(f'"{g}"' for g in r["groups"]) or '"unassigned"'
        body.append(f':{r["iri"]} :aclGroup {groups} ;')
        body.append(f'    :classification "{r["classification"]}" ;')
        body.append(f'    :docId "{r["doc_id"]}" .')

    STORE.write_text(text + "\n" + MARKER + "\n" + "\n".join(body) + "\n")
    return f"merged {len(rows)} documents into {STORE}"


def main() -> None:
    rows = extract()
    if not rows:
        raise SystemExit(f"No documents found under {CORPUS}")
    OUT.parent.mkdir(parents=True, exist_ok=True)
    OUT.write_text(render(rows))
    print(merge_into_store(rows))
    total_groups = sum(len(r["groups"]) for r in rows)
    missing = [r["doc_id"] for r in rows if not r["groups"]]
    print(f"{len(rows)} documents, {total_groups} group assignments -> {OUT}")
    for r in rows:
        print(f"  {r['doc_id']:10} {r['classification']:13} {', '.join(r['groups'])}")
    if missing:
        print(f"WARNING: no ACL group on {', '.join(missing)}")


if __name__ == "__main__":
    main()