Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

feat(kg): add RelationalSchemaMapper for row-and-column sources (#1640)

John Costaf6e75684d00a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0fbac0a6..d18f85a6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+
+- **`RelationalSchemaMapper`** (#1386, part 1) by @costajohnt
+ - `semantica.kg.RelationalSchemaMapper` maps rows from a relational source (`DBIngestor`, `SnowflakeIngestor`, `DatabricksIngestor`, `PandasIngestor`, a DataFrame or plain row dicts) to the `{"entities", "relationships"}` shape `GraphBuilder` and `OntologyGenerator` consume: entity tables become entities keyed by primary key, foreign keys become typed relationships, junction tables become relationships only
+ - Every entity and relationship is tagged with the `source` it came from so `ConflictDetector` can key credibility on it. New `tests/kg/test_schema_mapper.py`
+
## [0.7.0] - 2026-09-22
### Added
diff --git a/semantica/kg/__init__.py b/semantica/kg/__init__.py
index db0e6cba..c6288c34 100644
--- a/semantica/kg/__init__.py
+++ b/semantica/kg/__init__.py
@@ -134,6 +134,7 @@ from .path_finder import PathFinder
from .kg_provenance import GraphBuilderWithProvenance, AlgorithmTrackerWithProvenance
from .provenance_tracker import ProvenanceTracker
from .registry import MethodRegistry, method_registry, AlgorithmRegistry, algorithm_registry
+from .schema_mapper import RelationalSchemaMapper
from .seed_manager import SeedManager
from .similarity_calculator import SimilarityCalculator
from .temporal_query import (
@@ -148,6 +149,7 @@ from .temporal_query_rewriter import TemporalQueryRewriter, TemporalQueryResult
__all__ = [
# Core Classes
+ "RelationalSchemaMapper",
"KnowledgeGraph",
"HierarchicalCommunity",
"CommunityHierarchy",
diff --git a/semantica/kg/schema_mapper.py b/semantica/kg/schema_mapper.py
new file mode 100644
index 00000000..d179da7a
--- /dev/null
+++ b/semantica/kg/schema_mapper.py
@@ -0,0 +1,437 @@
+"""
+Relational schema mapper.
+
+Turns rows pulled from a relational source (``DBIngestor``, ``SnowflakeIngestor``,
+``DatabricksIngestor``, ``PandasIngestor``, a pandas ``DataFrame`` or plain lists
+of row dicts) into the ``{"entities": [...], "relationships": [...]}`` shape that
+``GraphBuilder`` and ``OntologyGenerator`` consume:
+
+- one row of an entity table becomes one entity; the primary key becomes the
+ entity id and the remaining columns become entity properties,
+- one foreign key becomes one typed relationship between the two entity ids,
+- a junction table (a table that is not an entity table and whose rows are held
+ together by two foreign keys) becomes relationships only, never a phantom
+ entity,
+- every entity and relationship records the ``source`` it came from so that
+ ``ConflictDetector`` / ``ConflictResolver`` and the provenance layer can key
+ credibility on it: entities carry it flat as ``source`` (where
+ ``ConflictDetector`` reads it) and both carry ``metadata: {"source", "table"}``
+ (``metadata`` is skipped by ``OntologyGenerator`` when it infers properties).
+
+A junction relationship runs from the first foreign key listed to the second
+(explicit keys come before schema-derived ones); its predicate is the second
+key's, then the first's, then the table name.
+
+Entity ids are ``"<Type>:<pk>"`` (``|`` and ``\\`` inside a key component are
+backslash-escaped), so the same row ingested from two systems of record maps to
+the same id and the two copies can be compared for conflicts.
+
+Example:
+ >>> mapper = RelationalSchemaMapper(
+ ... entity_tables={
+ ... "CUSTOMERS": {"pk": "CUSTOMER_ID", "type": "Customer", "name": "NAME"},
+ ... "ORDERS": {"pk": "ORDER_ID", "type": "Order"},
+ ... },
+ ... foreign_keys=[
+ ... {"table": "ORDERS", "column": "CUSTOMER_ID",
+ ... "references": ("CUSTOMERS", "CUSTOMER_ID"), "predicate": "placedBy"},
+ ... ],
+ ... )
+ >>> mapped = mapper.map({"CUSTOMERS": customers, "ORDERS": orders}, source="snowflake_crm")
+ >>> GraphBuilder().build(sources=[mapped])
+"""
+
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Union
+
+from ..utils.logging import get_logger
+
+Row = Dict[str, Any]
+TableInput = Union[Sequence[Row], Any]
+
+# Keys the mapper sets itself, plus the aliases GraphBuilder promotes to
+# "source" / "target" (a column named "object" would turn an entity into a
+# relationship). A column with one of these names is rejected, not overwritten.
+ENTITY_RESERVED = frozenset(
+ {
+ "id",
+ "entity_id",
+ "type",
+ "name",
+ "source",
+ "source_id",
+ "target",
+ "target_id",
+ "subject",
+ "object",
+ "metadata",
+ }
+)
+RELATIONSHIP_RESERVED = frozenset(
+ {
+ "source",
+ "source_id",
+ "target",
+ "target_id",
+ "subject",
+ "object",
+ "type",
+ "metadata",
+ }
+)
+
+
+class RelationalSchemaMapper:
+ """Map relational rows plus a small schema spec to entities and relationships.
+
+ Args:
+ entity_tables: ``{table_name: {"pk": column or [columns], "type": ClassName,
+ "name": column (optional)}}``. ``name`` names the column used as the
+ entity's display name; it defaults to the primary key value.
+ foreign_keys: Explicit foreign keys, ``[{"table", "column",
+ "references": (table, column), "predicate" (optional)}]``. This is the
+ required fallback for warehouses where constraints are informational
+ or absent. ``predicate`` defaults to the referenced table's name. The
+ referenced column must be that table's primary key (entity ids come
+ from it), so a key into any other column is a ``ValueError``.
+ schema: The dict returned by ``DBIngestor.get_database_schema()``; its
+ ``foreign_keys`` entries (SQLAlchemy inspector dicts) are used when no
+ explicit foreign key covers the same column. Constraints that point
+ at anything but an entity table's whole primary key are skipped with
+ a warning. The ingestor does not
+ record which table a constraint belongs to, so a ``table_name`` /
+ ``table`` key is used when present and the owning table is otherwise
+ inferred from which mapped table has the column.
+ """
+
+ def __init__(
+ self,
+ entity_tables: Mapping[str, Mapping[str, Any]],
+ foreign_keys: Optional[Iterable[Mapping[str, Any]]] = None,
+ schema: Optional[Mapping[str, Any]] = None,
+ ) -> None:
+ self.logger = get_logger("schema_mapper")
+ self.entity_tables: Dict[str, Dict[str, Any]] = {}
+ for table, spec in entity_tables.items():
+ if not spec.get("pk"):
+ raise ValueError(f"entity table {table!r} needs a 'pk' column")
+ if not spec.get("type"):
+ raise ValueError(f"entity table {table!r} needs a 'type' class name")
+ pk = spec["pk"]
+ self.entity_tables[table] = {
+ "pk": [pk] if isinstance(pk, str) else list(pk),
+ "type": spec["type"],
+ "name": spec.get("name"),
+ }
+
+ self._explicit: List[Dict[str, Any]] = [
+ self._normalize_foreign_key(fk) for fk in (foreign_keys or [])
+ ]
+ self._schema_fks: List[Dict[str, Any]] = [
+ fk for fk in self._foreign_keys_from_schema(schema) if fk is not None
+ ]
+
+ # ------------------------------------------------------------------ config
+
+ def _normalize_foreign_key(self, fk: Mapping[str, Any]) -> Dict[str, Any]:
+ table = fk.get("table")
+ column = fk.get("column")
+ references = fk.get("references")
+ if not table or not column or not references or len(references) != 2:
+ raise ValueError(
+ "foreign key needs 'table', 'column' and 'references': (table, column); "
+ f"got {dict(fk)!r}"
+ )
+ ref_table, ref_column = references
+ if ref_table not in self.entity_tables:
+ raise ValueError(
+ f"foreign key on {table}.{column} references {ref_table!r}, "
+ "which is not an entity table"
+ )
+ pk = self.entity_tables[ref_table]["pk"]
+ if [ref_column] != pk:
+ why = (
+ "a composite primary key cannot be referenced"
+ if len(pk) > 1
+ else "a foreign key must reference the primary key"
+ )
+ raise ValueError(
+ f"foreign key on {table}.{column} references {ref_table}.{ref_column}, "
+ f"but {ref_table!r} entity ids come from {pk}; {why}"
+ )
+ return {
+ "table": table,
+ "column": column,
+ "ref_table": ref_table,
+ "ref_column": ref_column,
+ "predicate": fk.get("predicate"), # None: defaults to the referenced table
+ }
+
+ def _foreign_keys_from_schema(
+ self, schema: Optional[Mapping[str, Any]]
+ ) -> Iterable[Optional[Dict[str, Any]]]:
+ for fk in (schema or {}).get("foreign_keys", []) or []:
+ columns = fk.get("constrained_columns") or []
+ ref_columns = fk.get("referred_columns") or []
+ ref_table = fk.get("referred_table")
+ # composite constraints have no single entity id to point at
+ if len(columns) != 1 or len(ref_columns) != 1 or not ref_table:
+ yield None
+ continue
+ if ref_table not in self.entity_tables:
+ yield None
+ continue
+ # a constraint on a unique non-key column has no entity id to point at
+ pk = self.entity_tables[ref_table]["pk"]
+ if list(ref_columns) != pk:
+ self.logger.warning(
+ "skipping schema foreign key %s.%s -> %s.%s: %r entity ids "
+ "come from %s",
+ fk.get("table_name") or fk.get("table") or "?",
+ columns[0],
+ ref_table,
+ ref_columns[0],
+ ref_table,
+ pk,
+ )
+ yield None
+ continue
+ yield {
+ "table": fk.get("table_name") or fk.get("table"),
+ "column": columns[0],
+ "ref_table": ref_table,
+ "ref_column": ref_columns[0],
+ "predicate": None,
+ }
+
+ def _foreign_keys_for(
+ self, table: str, columns: Iterable[str]
+ ) -> List[Dict[str, Any]]:
+ """Foreign keys that apply to ``table``: explicit ones first, then schema
+ ones for columns no explicit key already covers.
+
+ Inference by column name is a heuristic: a table that merely copies a
+ key column (a denormalised ``SHIPMENTS.CUSTOMER_ID``) is read as holding
+ that foreign key. Pass ``foreign_keys`` explicitly when that is wrong.
+ """
+ column_set = set(columns)
+ result = [fk for fk in self._explicit if fk["table"] == table]
+ covered = {fk["column"] for fk in result}
+ for fk in self._schema_fks:
+ owner = fk["table"]
+ if owner is not None and owner != table:
+ continue
+ if fk["column"] in covered:
+ continue
+ # a key recorded against this table is kept while the table is empty;
+ # once there are columns, recorded and inferred keys both need theirs
+ if fk["column"] not in column_set and (owner is None or column_set):
+ continue
+ # the referenced table carries that key column itself; without an
+ # owning table recorded, do not read it as a self-reference.
+ if (
+ owner is None
+ and table == fk["ref_table"]
+ and fk["column"] == fk["ref_column"]
+ ):
+ continue
+ result.append({**fk, "table": table})
+ covered.add(fk["column"])
+ return result
+
+ # ------------------------------------------------------------------ mapping
+
+ def map(
+ self, tables: Mapping[str, TableInput], source: str
+ ) -> Dict[str, List[Dict[str, Any]]]:
+ """Map ``{table_name: rows}`` to entities and relationships.
+
+ ``rows`` may be a list of dicts, an ingestor result exposing ``.data`` or
+ ``.rows`` (Snowflake, Databricks, DBIngestor) or ``.dataframe``
+ (PandasIngestor), one of the ``{"columns", "row_count", "rows"}``
+ dicts in ``DBIngestor.ingest_database()["tables"]``, or a pandas
+ DataFrame. Tables listed in
+ ``entity_tables`` produce entities; other tables must be junction tables
+ held together by exactly two foreign keys and produce relationships only.
+ """
+ entities: List[Dict[str, Any]] = []
+ relationships: List[Dict[str, Any]] = []
+
+ for table, raw in tables.items():
+ rows = _rows_of(raw)
+ columns = _columns_of(rows)
+ foreign_keys = self._foreign_keys_for(table, columns)
+
+ if table in self.entity_tables:
+ for row in rows:
+ entity = self._entity(table, row, source)
+ if entity is None:
+ continue
+ entities.append(entity)
+ relationships.extend(
+ self._fk_relationships(
+ table, row, entity["id"], foreign_keys, source
+ )
+ )
+ elif len(foreign_keys) == 2:
+ relationships.extend(
+ self._junction_relationships(table, rows, foreign_keys, source)
+ )
+ else:
+ raise ValueError(
+ f"table {table!r} is not an entity table and has {len(foreign_keys)} "
+ "foreign key(s); a junction table needs exactly two"
+ )
+
+ return {"entities": entities, "relationships": relationships}
+
+ def _entity(self, table: str, row: Row, source: str) -> Optional[Dict[str, Any]]:
+ spec = self.entity_tables[table]
+ key_values = [row.get(column) for column in spec["pk"]]
+ if any(_is_missing(value) for value in key_values):
+ return None
+ pk = _pk_text(key_values)
+ name_column = spec["name"]
+ name = row.get(name_column) if name_column else None
+ entity: Dict[str, Any] = {
+ "id": f"{spec['type']}:{pk}",
+ "type": spec["type"],
+ "name": str(name) if not _is_missing(name) else pk,
+ }
+ for column, value in row.items():
+ if column in spec["pk"]:
+ continue
+ if column in ENTITY_RESERVED:
+ if column == name_column:
+ continue # already the entity's name
+ raise ValueError(
+ f"column {column!r} in table {table!r} clashes with a reserved "
+ f"entity key; rename it (reserved: {sorted(ENTITY_RESERVED)})"
+ )
+ entity[column] = value
+ entity["source"] = source
+ entity["metadata"] = {"source": source, "table": table}
+ return entity
+
+ def _fk_relationships(
+ self,
+ table: str,
+ row: Row,
+ entity_id: str,
+ foreign_keys: Sequence[Dict[str, Any]],
+ source: str,
+ ) -> Iterable[Dict[str, Any]]:
+ for fk in foreign_keys:
+ target = self._referenced_id(fk, row)
+ if target is None:
+ continue
+ yield {
+ "source": entity_id,
+ "target": target,
+ "type": fk["predicate"] or fk["ref_table"],
+ "metadata": {"source": source, "table": table},
+ }
+
+ def _junction_relationships(
+ self,
+ table: str,
+ rows: Sequence[Row],
+ foreign_keys: Sequence[Dict[str, Any]],
+ source: str,
+ ) -> Iterable[Dict[str, Any]]:
+ first, second = foreign_keys
+ # an explicit predicate on either key names the relationship; the
+ # second key's predicate wins because it names the target side.
+ predicate = second["predicate"] or first["predicate"] or table
+ key_columns = {first["column"], second["column"]}
+ for row in rows:
+ head = self._referenced_id(first, row)
+ tail = self._referenced_id(second, row)
+ if head is None or tail is None:
+ continue
+ relationship: Dict[str, Any] = {
+ "source": head,
+ "target": tail,
+ "type": predicate,
+ }
+ for column, value in row.items():
+ if column in key_columns:
+ continue
+ if column in RELATIONSHIP_RESERVED:
+ raise ValueError(
+ f"column {column!r} in junction table {table!r} clashes with a "
+ f"reserved relationship key; rename it "
+ f"(reserved: {sorted(RELATIONSHIP_RESERVED)})"
+ )
+ relationship[column] = value
+ relationship["metadata"] = {"source": source, "table": table}
+ yield relationship
+
+ def _referenced_id(self, fk: Dict[str, Any], row: Row) -> Optional[str]:
+ value = row.get(fk["column"])
+ if _is_missing(value):
+ return None
+ return f"{self.entity_tables[fk['ref_table']]['type']}:{_pk_text([value])}"
+
+
+# ---------------------------------------------------------------------- helpers
+
+
+def _rows_of(raw: TableInput) -> List[Row]:
+ """Accept row dicts, ingestor results, or a DataFrame."""
+ if isinstance(raw, (list, tuple)):
+ return [_without_missing(dict(row)) for row in raw]
+ if isinstance(raw, Mapping) and isinstance(raw.get("rows"), (list, tuple)):
+ return [_without_missing(dict(row)) for row in raw["rows"]]
+ for attribute in ("data", "rows"):
+ value = getattr(raw, attribute, None)
+ if isinstance(value, (list, tuple)):
+ return [_without_missing(dict(row)) for row in value]
+ frame = getattr(raw, "dataframe", raw)
+ if hasattr(frame, "to_dict"):
+ return [_without_missing(row) for row in frame.to_dict("records")]
+ raise TypeError(f"unsupported table input {type(raw).__name__}")
+
+
+def _without_missing(row: Row) -> Row:
+ """pandas marks gaps as NaN / NaT / NA; the graph wants plain None."""
+ return {key: (None if _is_missing(value) else value) for key, value in row.items()}
+
+
+def _is_missing(value: Any) -> bool:
+ if value is None:
+ return True
+ try:
+ import pandas as pd
+
+ return bool(pd.isna(value))
+ except (TypeError, ValueError):
+ # lists and other non-scalars
+ return False
+
+
+def _pk_text(values: Iterable[Any]) -> str:
+ """Join key components so that ``|`` and ``\\`` inside a component cannot
+ make two keys read as one; used for entity ids and relationship targets
+ alike so the two always agree."""
+ return "|".join(
+ _key_text(value).replace("\\", "\\\\").replace("|", "\\|") for value in values
+ )
+
+
+def _key_text(value: Any) -> str:
+ """Key text that is the same whichever source the row came from.
+
+ A pandas integer column with a gap is read back as float64, so a key of
+ ``1`` arrives as ``1.0``; that must still map to ``"1"``.
+ """
+ if isinstance(value, float) and value.is_integer():
+ return str(int(value))
+ return str(value)
+
+
+def _columns_of(rows: Sequence[Row]) -> List[str]:
+ seen: Dict[str, None] = {}
+ for row in rows:
+ for column in row:
+ seen.setdefault(column, None)
+ return list(seen)
diff --git a/tests/kg/test_schema_mapper.py b/tests/kg/test_schema_mapper.py
new file mode 100644
index 00000000..ae197036
--- /dev/null
+++ b/tests/kg/test_schema_mapper.py
@@ -0,0 +1,705 @@
+"""Tests for RelationalSchemaMapper (issue #1386, part 1)."""
+
+import logging
+import math
+from types import SimpleNamespace
+
+import pandas as pd
+import pytest
+
+from semantica.kg import RelationalSchemaMapper
+from semantica.kg.schema_mapper import RelationalSchemaMapper as DirectImport
+
+CUSTOMERS = [
+ {"CUSTOMER_ID": 1, "NAME": "Acme", "ADDRESS": "1 Main St", "CREDIT_LIMIT": 1000},
+ {"CUSTOMER_ID": 2, "NAME": "Globex", "ADDRESS": "2 Side St", "CREDIT_LIMIT": 500},
+]
+ORDERS = [
+ {"ORDER_ID": 10, "CUSTOMER_ID": 1, "AMOUNT": 99.5},
+ {"ORDER_ID": 11, "CUSTOMER_ID": 2, "AMOUNT": 12.0},
+ {"ORDER_ID": 12, "CUSTOMER_ID": None, "AMOUNT": 1.0},
+]
+PRODUCTS = [
+ {"PRODUCT_ID": "p1", "SKU": "SKU-1", "CATEGORY": "Electronics"},
+ {"PRODUCT_ID": "p2", "SKU": "SKU-2", "CATEGORY": "Electronics"},
+]
+ORDER_ITEMS = [
+ {"ORDER_ID": 10, "PRODUCT_ID": "p1", "QUANTITY": 2},
+ {"ORDER_ID": 10, "PRODUCT_ID": "p2", "QUANTITY": 1},
+ {"ORDER_ID": 11, "PRODUCT_ID": "p2", "QUANTITY": 3},
+]
+
+ENTITY_TABLES = {
+ "CUSTOMERS": {"pk": "CUSTOMER_ID", "type": "Customer", "name": "NAME"},
+ "ORDERS": {"pk": "ORDER_ID", "type": "Order"},
+ "PRODUCTS": {"pk": "PRODUCT_ID", "type": "Product", "name": "SKU"},
+}
+FOREIGN_KEYS = [
+ {
+ "table": "ORDERS",
+ "column": "CUSTOMER_ID",
+ "references": ("CUSTOMERS", "CUSTOMER_ID"),
+ "predicate": "placedBy",
+ },
+ {
+ "table": "ORDER_ITEMS",
+ "column": "ORDER_ID",
+ "references": ("ORDERS", "ORDER_ID"),
+ },
+ {
+ "table": "ORDER_ITEMS",
+ "column": "PRODUCT_ID",
+ "references": ("PRODUCTS", "PRODUCT_ID"),
+ "predicate": "containsProduct",
+ },
+]
+
+
+def by_id(entities):
+ return {entity["id"]: entity for entity in entities}
+
+
+def test_exported_from_semantica_kg():
+ assert RelationalSchemaMapper is DirectImport
+
+
+def test_rows_become_entities_with_flat_properties_and_source():
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+
+ mapped = mapper.map({"CUSTOMERS": CUSTOMERS}, source="snowflake_crm")
+
+ assert set(mapped) == {"entities", "relationships"}
+ assert mapped["relationships"] == []
+ entities = by_id(mapped["entities"])
+ assert set(entities) == {"Customer:1", "Customer:2"}
+ acme = entities["Customer:1"]
+ assert acme["type"] == "Customer"
+ assert acme["name"] == "Acme"
+ # non-key columns land flat on the entity so ConflictDetector can read them
+ assert acme["ADDRESS"] == "1 Main St"
+ assert acme["CREDIT_LIMIT"] == 1000
+ assert "CUSTOMER_ID" not in acme
+ assert acme["source"] == "snowflake_crm"
+ assert acme["metadata"] == {"source": "snowflake_crm", "table": "CUSTOMERS"}
+
+
+def test_same_row_from_two_systems_shares_an_id():
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+ crm = mapper.map({"CUSTOMERS": CUSTOMERS}, source="snowflake_crm")
+ master = mapper.map(
+ {"CUSTOMERS": [{"CUSTOMER_ID": 1, "NAME": "Acme", "ADDRESS": "1 Main Street"}]},
+ source="databricks_master",
+ )
+
+ ids = {e["id"] for e in crm["entities"]} & {e["id"] for e in master["entities"]}
+ assert ids == {"Customer:1"}
+ sources = {e["source"] for e in crm["entities"] + master["entities"]}
+ assert sources == {"snowflake_crm", "databricks_master"}
+
+
+def test_foreign_keys_become_typed_relationships():
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS
+ )
+
+ mapped = mapper.map({"CUSTOMERS": CUSTOMERS, "ORDERS": ORDERS}, source="crm")
+
+ orders = by_id(mapped["entities"])
+ assert orders["Order:10"]["AMOUNT"] == 99.5
+ assert "CUSTOMER_ID" in orders["Order:10"], "FK column stays a property too"
+ rels = mapped["relationships"]
+ assert rels == [
+ {
+ "source": "Order:10",
+ "target": "Customer:1",
+ "type": "placedBy",
+ "metadata": {"source": "crm", "table": "ORDERS"},
+ },
+ {
+ "source": "Order:11",
+ "target": "Customer:2",
+ "type": "placedBy",
+ "metadata": {"source": "crm", "table": "ORDERS"},
+ },
+ ]
+ # a null FK produces no relationship and no error
+
+
+def test_junction_table_maps_to_relationships_only():
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS
+ )
+
+ mapped = mapper.map(
+ {"ORDERS": ORDERS, "PRODUCTS": PRODUCTS, "ORDER_ITEMS": ORDER_ITEMS},
+ source="crm",
+ )
+
+ assert not [
+ e for e in mapped["entities"] if e["metadata"]["table"] == "ORDER_ITEMS"
+ ]
+ junction = [
+ r for r in mapped["relationships"] if r["metadata"]["table"] == "ORDER_ITEMS"
+ ]
+ assert junction == [
+ {
+ "source": "Order:10",
+ "target": "Product:p1",
+ "type": "containsProduct",
+ "QUANTITY": 2,
+ "metadata": {"source": "crm", "table": "ORDER_ITEMS"},
+ },
+ {
+ "source": "Order:10",
+ "target": "Product:p2",
+ "type": "containsProduct",
+ "QUANTITY": 1,
+ "metadata": {"source": "crm", "table": "ORDER_ITEMS"},
+ },
+ {
+ "source": "Order:11",
+ "target": "Product:p2",
+ "type": "containsProduct",
+ "QUANTITY": 3,
+ "metadata": {"source": "crm", "table": "ORDER_ITEMS"},
+ },
+ ]
+
+
+def test_junction_predicate_defaults_to_table_name_when_not_given():
+ fks = [
+ {
+ "table": "ORDER_ITEMS",
+ "column": "ORDER_ID",
+ "references": ("ORDERS", "ORDER_ID"),
+ },
+ {
+ "table": "ORDER_ITEMS",
+ "column": "PRODUCT_ID",
+ "references": ("PRODUCTS", "PRODUCT_ID"),
+ },
+ ]
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, foreign_keys=fks)
+
+ mapped = mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="crm")
+
+ assert mapped["relationships"][0]["type"] == "ORDER_ITEMS"
+
+
+def test_foreign_keys_from_db_ingestor_schema():
+ # The shape DBIngestor.get_database_schema()["foreign_keys"] returns
+ # (SQLAlchemy inspector dicts). The ingestor does not record the owning
+ # table, so it is inferred from which provided table has the column.
+ schema = {
+ "tables": ["CUSTOMERS", "ORDERS"],
+ "foreign_keys": [
+ {
+ "name": "fk_orders_customer",
+ "constrained_columns": ["CUSTOMER_ID"],
+ "referred_table": "CUSTOMERS",
+ "referred_columns": ["CUSTOMER_ID"],
+ "referred_schema": None,
+ }
+ ],
+ }
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)
+
+ mapped = mapper.map({"CUSTOMERS": CUSTOMERS, "ORDERS": ORDERS}, source="pg")
+
+ rels = mapped["relationships"]
+ assert [(r["source"], r["target"], r["type"]) for r in rels] == [
+ ("Order:10", "Customer:1", "CUSTOMERS"),
+ ("Order:11", "Customer:2", "CUSTOMERS"),
+ ]
+
+
+def test_schema_foreign_keys_use_table_name_when_present():
+ schema = {
+ "foreign_keys": [
+ {
+ "table_name": "ORDERS",
+ "constrained_columns": ["CUSTOMER_ID"],
+ "referred_table": "CUSTOMERS",
+ "referred_columns": ["CUSTOMER_ID"],
+ }
+ ]
+ }
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)
+ mapped = mapper.map({"ORDERS": ORDERS[:1]}, source="pg")
+ assert mapped["relationships"] == [
+ {
+ "source": "Order:10",
+ "target": "Customer:1",
+ "type": "CUSTOMERS",
+ "metadata": {"source": "pg", "table": "ORDERS"},
+ }
+ ]
+
+
+def test_explicit_foreign_keys_override_schema_predicates():
+ schema = {
+ "foreign_keys": [
+ {
+ "table_name": "ORDERS",
+ "constrained_columns": ["CUSTOMER_ID"],
+ "referred_table": "CUSTOMERS",
+ "referred_columns": ["CUSTOMER_ID"],
+ }
+ ]
+ }
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS[:1], schema=schema
+ )
+ mapped = mapper.map({"ORDERS": ORDERS[:1]}, source="pg")
+ assert [r["type"] for r in mapped["relationships"]] == ["placedBy"]
+
+
+ "wrap",
+ [
+ lambda rows: SimpleNamespace(data=rows), # SnowflakeData / DatabricksData
+ lambda rows: SimpleNamespace(rows=rows), # TableData (DBIngestor)
+ # DBIngestor.ingest_database()["tables"][name]
+ lambda rows: {"columns": list(rows[0]), "row_count": len(rows), "rows": rows},
+ lambda rows: SimpleNamespace(dataframe=pd.DataFrame(rows)), # PandasData
+ lambda rows: pd.DataFrame(rows),
+ ],
+)
+def test_accepts_ingestor_results_and_dataframes(wrap):
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+
+ mapped = mapper.map({"CUSTOMERS": wrap(CUSTOMERS)}, source="x")
+
+ assert [e["id"] for e in mapped["entities"]] == ["Customer:1", "Customer:2"]
+ assert mapped["entities"][0]["ADDRESS"] == "1 Main St"
+
+
+def test_dataframe_gaps_become_none():
+ frame = pd.DataFrame(
+ [
+ {
+ "CUSTOMER_ID": 1,
+ "CREDIT_LIMIT": 100,
+ "SINCE": pd.Timestamp("2020-01-01"),
+ },
+ {"CUSTOMER_ID": 2, "CREDIT_LIMIT": None, "SINCE": None},
+ ]
+ )
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+
+ entities = mapper.map({"CUSTOMERS": frame}, source="x")["entities"]
+
+ assert entities[0]["CREDIT_LIMIT"] == 100
+ assert entities[1]["CREDIT_LIMIT"] is None # float NaN column
+ assert entities[1]["SINCE"] is None # NaT
+ assert not any(isinstance(v, float) and math.isnan(v) for v in entities[1].values())
+
+
+def test_dataframe_integer_keys_with_gaps_keep_stable_ids():
+ # an int column with a missing row is read back as float64 (1.0, NaN)
+ frame = pd.DataFrame(
+ [{"CUSTOMER_ID": 1, "NAME": "Acme"}, {"CUSTOMER_ID": None, "NAME": "x"}]
+ )
+ orders = pd.DataFrame(
+ [{"ORDER_ID": 10, "CUSTOMER_ID": 1}, {"ORDER_ID": 11, "CUSTOMER_ID": None}]
+ )
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS[:1]
+ )
+
+ mapped = mapper.map({"CUSTOMERS": frame, "ORDERS": orders}, source="df")
+
+ assert [e["id"] for e in mapped["entities"]] == [
+ "Customer:1",
+ "Order:10",
+ "Order:11",
+ ]
+ assert [(r["source"], r["target"]) for r in mapped["relationships"]] == [
+ ("Order:10", "Customer:1")
+ ]
+ plain = mapper.map({"CUSTOMERS": CUSTOMERS[:1]}, source="rows")["entities"][0]["id"]
+ assert plain == "Customer:1"
+
+
+def test_reserved_columns_are_rejected():
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS
+ )
+ with pytest.raises(ValueError, match="reserved entity key"):
+ mapper.map({"CUSTOMERS": [{"CUSTOMER_ID": 1, "type": "gold"}]}, source="x")
+ # GraphBuilder promotes "object"/"subject"/"target_id" to relationship ends
+ with pytest.raises(ValueError, match="reserved entity key"):
+ mapper.map({"CUSTOMERS": [{"CUSTOMER_ID": 1, "object": "x"}]}, source="x")
+ with pytest.raises(ValueError, match="reserved relationship key"):
+ mapper.map(
+ {"ORDER_ITEMS": [{"ORDER_ID": 10, "PRODUCT_ID": "p1", "target": "z"}]},
+ source="x",
+ )
+ # upper-case ERP columns do not clash
+ mapped = mapper.map({"CUSTOMERS": [{"CUSTOMER_ID": 1, "TYPE": "gold"}]}, source="x")
+ assert mapped["entities"][0]["TYPE"] == "gold"
+ assert mapped["entities"][0]["type"] == "Customer"
+
+
+def test_junction_predicate_precedence():
+ def fks(first_pred, second_pred):
+ return [
+ {
+ "table": "ORDER_ITEMS",
+ "column": "ORDER_ID",
+ "references": ("ORDERS", "ORDER_ID"),
+ **({"predicate": first_pred} if first_pred else {}),
+ },
+ {
+ "table": "ORDER_ITEMS",
+ "column": "PRODUCT_ID",
+ "references": ("PRODUCTS", "PRODUCT_ID"),
+ **({"predicate": second_pred} if second_pred else {}),
+ },
+ ]
+
+ def predicate(first_pred, second_pred):
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=fks(first_pred, second_pred)
+ )
+ return mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")[
+ "relationships"
+ ][0]["type"]
+
+ assert predicate("hasItem", "containsProduct") == "containsProduct"
+ assert predicate("hasItem", None) == "hasItem"
+ assert predicate(None, None) == "ORDER_ITEMS"
+ # a predicate that happens to equal the referenced table name is still explicit
+ assert predicate(None, "PRODUCTS") == "PRODUCTS"
+
+
+def test_junction_edge_cases():
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS
+ )
+ # a null key in a junction row is skipped
+ rows = [{"ORDER_ID": 10, "PRODUCT_ID": None, "QUANTITY": 1}]
+ assert mapper.map({"ORDER_ITEMS": rows}, source="x")["relationships"] == []
+
+ # three foreign keys is not a junction table
+ three = FOREIGN_KEYS + [
+ {
+ "table": "ORDER_ITEMS",
+ "column": "CUSTOMER_ID",
+ "references": ("CUSTOMERS", "CUSTOMER_ID"),
+ }
+ ]
+ mapper3 = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, foreign_keys=three)
+ with pytest.raises(ValueError, match="3 foreign key"):
+ mapper3.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")
+
+ # a junction table described only by the DB schema
+ schema = {
+ "foreign_keys": [
+ {
+ "table_name": "ORDER_ITEMS",
+ "constrained_columns": ["ORDER_ID"],
+ "referred_table": "ORDERS",
+ "referred_columns": ["ORDER_ID"],
+ },
+ {
+ "table_name": "ORDER_ITEMS",
+ "constrained_columns": ["PRODUCT_ID"],
+ "referred_table": "PRODUCTS",
+ "referred_columns": ["PRODUCT_ID"],
+ },
+ ]
+ }
+ schema_mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)
+ rel = schema_mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")[
+ "relationships"
+ ][0]
+ assert (rel["source"], rel["target"], rel["type"]) == (
+ "Order:10",
+ "Product:p1",
+ "ORDER_ITEMS",
+ )
+
+
+def test_schema_inference_self_reference_and_limits():
+ employees = {"EMPLOYEES": {"pk": "EMP_ID", "type": "Employee"}}
+ schema = {
+ "foreign_keys": [
+ {
+ "constrained_columns": ["MANAGER_ID"],
+ "referred_table": "EMPLOYEES",
+ "referred_columns": ["EMP_ID"],
+ },
+ # composite constraints have no single entity id: ignored
+ {
+ "constrained_columns": ["A", "B"],
+ "referred_table": "EMPLOYEES",
+ "referred_columns": ["EMP_ID", "X"],
+ },
+ ]
+ }
+ mapper = RelationalSchemaMapper(entity_tables=employees, schema=schema)
+ rows = [{"EMP_ID": 1, "MANAGER_ID": None}, {"EMP_ID": 2, "MANAGER_ID": 1}]
+
+ rels = mapper.map({"EMPLOYEES": rows}, source="hr")["relationships"]
+
+ assert [(r["source"], r["target"], r["type"]) for r in rels] == [
+ ("Employee:2", "Employee:1", "EMPLOYEES")
+ ]
+
+ # documented limit: inference by column name also matches a denormalised copy
+ tables = {**ENTITY_TABLES, "SHIPMENTS": {"pk": "SHIPMENT_ID", "type": "Shipment"}}
+ fk_schema = {
+ "foreign_keys": [
+ {
+ "constrained_columns": ["CUSTOMER_ID"],
+ "referred_table": "CUSTOMERS",
+ "referred_columns": ["CUSTOMER_ID"],
+ }
+ ]
+ }
+ shipments = [{"SHIPMENT_ID": "s1", "CUSTOMER_ID": 1}]
+ rels = RelationalSchemaMapper(entity_tables=tables, schema=fk_schema).map(
+ {"SHIPMENTS": shipments}, source="x"
+ )["relationships"]
+ assert [(r["source"], r["target"]) for r in rels] == [("Shipment:s1", "Customer:1")]
+
+
+def test_composite_key_components_containing_the_separator_do_not_collide():
+ mapper = RelationalSchemaMapper(
+ entity_tables={"P": {"pk": ["A", "B"], "type": "Pair"}}
+ )
+ rows = [
+ {"A": "x|y", "B": "z"},
+ {"A": "x", "B": "y|z"},
+ {"A": "x\\", "B": "y|z"},
+ {"A": "x|y\\", "B": "z"},
+ ]
+ ids = [e["id"] for e in mapper.map({"P": rows}, source="x")["entities"]]
+ assert len(set(ids)) == 4
+
+
+def test_relationship_targets_use_the_same_key_encoding_as_entity_ids():
+ mapper = RelationalSchemaMapper(
+ entity_tables={
+ "C": {"pk": "CODE", "type": "C"},
+ "O": {"pk": "ID", "type": "O"},
+ },
+ foreign_keys=[{"table": "O", "column": "CODE", "references": ("C", "CODE")}],
+ )
+ mapped = mapper.map(
+ {"C": [{"CODE": "a\\|b"}], "O": [{"ID": 1, "CODE": "a\\|b"}]}, source="x"
+ )
+ ids = {e["id"] for e in mapped["entities"]}
+ assert mapped["relationships"][0]["target"] in ids
+
+
+def test_metadata_is_skipped_by_the_ontology_property_generator():
+ from semantica.ontology.property_generator import PropertyGenerator
+
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+ entity = mapper.map({"ORDERS": ORDERS[:1]}, source="crm")["entities"][0]
+
+ properties = PropertyGenerator()._extract_data_properties([entity])
+
+ assert "AMOUNT" in properties
+ assert "metadata" not in properties
+ assert "table" not in properties
+
+
+def test_junction_direction_follows_foreign_key_order():
+ reversed_fks = [
+ FOREIGN_KEYS[2],
+ FOREIGN_KEYS[1],
+ ] # PRODUCT_ID first, ORDER_ID second
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=reversed_fks
+ )
+ rel = mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")["relationships"][0]
+ assert (rel["source"], rel["target"], rel["type"]) == (
+ "Product:p1",
+ "Order:10",
+ "containsProduct",
+ )
+
+
+def test_display_column_named_name_is_allowed():
+ mapper = RelationalSchemaMapper(
+ entity_tables={"customers": {"pk": "id", "type": "Customer", "name": "name"}}
+ )
+ entity = mapper.map(
+ {"customers": [{"id": 1, "name": "Acme", "city": "SF"}]}, source="pg"
+ )["entities"][0]
+ assert entity == {
+ "id": "Customer:1",
+ "type": "Customer",
+ "name": "Acme",
+ "city": "SF",
+ "source": "pg",
+ "metadata": {"source": "pg", "table": "customers"},
+ }
+
+
+def test_gaps_in_plain_rows_become_none_too():
+ rows = [{"CUSTOMER_ID": 1, "ADDRESS": float("nan"), "SINCE": pd.NaT, "TAGS": ["a"]}]
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+ entity = mapper.map({"CUSTOMERS": rows}, source="x")["entities"][0]
+ assert entity["ADDRESS"] is None
+ assert entity["SINCE"] is None
+ assert entity["TAGS"] == ["a"]
+
+
+def test_composite_key_with_a_missing_component_is_skipped():
+ mapper = RelationalSchemaMapper(
+ entity_tables={"LINES": {"pk": ["ORDER_ID", "LINE_NO"], "type": "OrderLine"}}
+ )
+ rows = [{"ORDER_ID": 10, "LINE_NO": None}, {"ORDER_ID": 10, "LINE_NO": 2}]
+ assert [e["id"] for e in mapper.map({"LINES": rows}, source="x")["entities"]] == [
+ "OrderLine:10|2"
+ ]
+
+
+def test_composite_primary_key_and_name_fallback():
+ mapper = RelationalSchemaMapper(
+ entity_tables={"LINES": {"pk": ["ORDER_ID", "LINE_NO"], "type": "OrderLine"}}
+ )
+ rows = [{"ORDER_ID": 10, "LINE_NO": 1, "QTY": 2}]
+
+ entity = mapper.map({"LINES": rows}, source="x")["entities"][0]
+
+ assert entity["id"] == "OrderLine:10|1"
+ assert entity["name"] == "10|1"
+ assert entity["QTY"] == 2
+ assert "ORDER_ID" not in entity and "LINE_NO" not in entity
+
+
+def test_rows_without_a_primary_key_are_skipped():
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+ rows = [{"CUSTOMER_ID": None, "NAME": "ghost"}, {"CUSTOMER_ID": 3, "NAME": "real"}]
+
+ mapped = mapper.map({"CUSTOMERS": rows}, source="x")
+
+ assert [e["id"] for e in mapped["entities"]] == ["Customer:3"]
+
+
+def test_unknown_table_without_foreign_keys_is_an_error():
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)
+
+ with pytest.raises(ValueError, match="MYSTERY"):
+ mapper.map({"MYSTERY": [{"A": 1}]}, source="x")
+
+
+def test_config_validation():
+ with pytest.raises(ValueError, match="pk"):
+ RelationalSchemaMapper(entity_tables={"T": {"type": "Thing"}})
+ with pytest.raises(ValueError, match="type"):
+ RelationalSchemaMapper(entity_tables={"T": {"pk": "ID"}})
+ with pytest.raises(ValueError, match="references"):
+ RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES,
+ foreign_keys=[{"table": "ORDERS", "column": "CUSTOMER_ID"}],
+ )
+ with pytest.raises(ValueError, match="entity table"):
+ RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES,
+ foreign_keys=[
+ {
+ "table": "ORDERS",
+ "column": "CUSTOMER_ID",
+ "references": ("NOPE", "ID"),
+ }
+ ],
+ )
+ # entity ids come from the primary key, so a key into any other column
+ # would point at an id that no row produces
+ with pytest.raises(ValueError, match="primary key"):
+ RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES,
+ foreign_keys=[
+ {
+ "table": "ORDERS",
+ "column": "CUSTOMER_EMAIL",
+ "references": ("CUSTOMERS", "EMAIL"),
+ }
+ ],
+ )
+ with pytest.raises(ValueError, match="composite"):
+ RelationalSchemaMapper(
+ entity_tables={"LINES": {"pk": ["ORDER_ID", "LINE_NO"], "type": "Line"}},
+ foreign_keys=[
+ {
+ "table": "SHIPMENTS",
+ "column": "ORDER_ID",
+ "references": ("LINES", "ORDER_ID"),
+ }
+ ],
+ )
+
+
+def test_schema_foreign_keys_to_non_primary_columns_are_skipped(caplog):
+ schema = {
+ "foreign_keys": [
+ {
+ "constrained_columns": ["CUSTOMER_EMAIL"],
+ "referred_table": "CUSTOMERS",
+ "referred_columns": ["EMAIL"],
+ }
+ ]
+ }
+ with caplog.at_level(logging.WARNING, logger="semantica.schema_mapper"):
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)
+ orders = [{"ORDER_ID": 10, "CUSTOMER_EMAIL": "a@x"}]
+
+ mapped = mapper.map({"ORDERS": orders}, source="pg")
+
+ assert mapped["relationships"] == []
+ assert any("CUSTOMERS.EMAIL" in r.getMessage() for r in caplog.records)
+
+
+def test_empty_junction_table_with_schema_foreign_keys_maps_to_nothing():
+ schema = {
+ "foreign_keys": [
+ {
+ "table_name": "ORDER_ITEMS",
+ "constrained_columns": ["ORDER_ID"],
+ "referred_table": "ORDERS",
+ "referred_columns": ["ORDER_ID"],
+ },
+ {
+ "table_name": "ORDER_ITEMS",
+ "constrained_columns": ["PRODUCT_ID"],
+ "referred_table": "PRODUCTS",
+ "referred_columns": ["PRODUCT_ID"],
+ },
+ ]
+ }
+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)
+
+ assert mapper.map({"ORDER_ITEMS": []}, source="pg") == {
+ "entities": [],
+ "relationships": [],
+ }
+ # rows that exist but lack a declared key column are still a mismatch
+ with pytest.raises(ValueError, match="1 foreign key"):
+ mapper.map({"ORDER_ITEMS": [{"ORDER_ID": 10, "QUANTITY": 1}]}, source="pg")
+
+
+def test_output_feeds_graph_builder():
+ from semantica.kg import GraphBuilder
+
+ mapper = RelationalSchemaMapper(
+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS
+ )
+ mapped = mapper.map({"CUSTOMERS": CUSTOMERS, "ORDERS": ORDERS}, source="crm")
+
+ graph = GraphBuilder().build(sources=[mapped])
+
+ entities = by_id(graph["entities"])
+ assert {"Customer:1", "Customer:2", "Order:10", "Order:11", "Order:12"} <= set(
+ entities
+ )
+ assert entities["Customer:1"]["ADDRESS"] == "1 Main St"
+ assert entities["Customer:1"]["source"] == "crm"
+ edges = {(r["source"], r["target"], r["type"]) for r in graph["relationships"]}
+ assert ("Order:10", "Customer:1", "placedBy") in edges
+ assert not any(target.endswith(":None") for _, target, _ in edges)