Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
feat(kg): add RelationalSchemaMapper for row-and-column sources (#1640)
John Costaf6e75684d00a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 0fbac0a6..d18f85a6 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]+### Added++- **`RelationalSchemaMapper`** (#1386, part 1) by @costajohnt+ - `semantica.kg.RelationalSchemaMapper` maps rows from a relational source (`DBIngestor`, `SnowflakeIngestor`, `DatabricksIngestor`, `PandasIngestor`, a DataFrame or plain row dicts) to the `{"entities", "relationships"}` shape `GraphBuilder` and `OntologyGenerator` consume: entity tables become entities keyed by primary key, foreign keys become typed relationships, junction tables become relationships only+ - Every entity and relationship is tagged with the `source` it came from so `ConflictDetector` can key credibility on it. New `tests/kg/test_schema_mapper.py`+## [0.7.0] - 2026-09-22### Addeddiff --git a/semantica/kg/__init__.py b/semantica/kg/__init__.pyindex db0e6cba..c6288c34 100644--- a/semantica/kg/__init__.py+++ b/semantica/kg/__init__.py@@ -134,6 +134,7 @@ from .path_finder import PathFinderfrom .kg_provenance import GraphBuilderWithProvenance, AlgorithmTrackerWithProvenancefrom .provenance_tracker import ProvenanceTrackerfrom .registry import MethodRegistry, method_registry, AlgorithmRegistry, algorithm_registry+from .schema_mapper import RelationalSchemaMapperfrom .seed_manager import SeedManagerfrom .similarity_calculator import SimilarityCalculatorfrom .temporal_query import (@@ -148,6 +149,7 @@ from .temporal_query_rewriter import TemporalQueryRewriter, TemporalQueryResult__all__ = [# Core Classes+ "RelationalSchemaMapper","KnowledgeGraph","HierarchicalCommunity","CommunityHierarchy",diff --git a/semantica/kg/schema_mapper.py b/semantica/kg/schema_mapper.pynew file mode 100644index 00000000..d179da7a--- /dev/null+++ b/semantica/kg/schema_mapper.py@@ -0,0 +1,437 @@+"""+Relational schema mapper.++Turns rows pulled from a relational source (``DBIngestor``, ``SnowflakeIngestor``,+``DatabricksIngestor``, ``PandasIngestor``, a pandas ``DataFrame`` or plain lists+of row dicts) into the ``{"entities": [...], "relationships": [...]}`` shape that+``GraphBuilder`` and ``OntologyGenerator`` consume:++- one row of an entity table becomes one entity; the primary key becomes the+ entity id and the remaining columns become entity properties,+- one foreign key becomes one typed relationship between the two entity ids,+- a junction table (a table that is not an entity table and whose rows are held+ together by two foreign keys) becomes relationships only, never a phantom+ entity,+- every entity and relationship records the ``source`` it came from so that+ ``ConflictDetector`` / ``ConflictResolver`` and the provenance layer can key+ credibility on it: entities carry it flat as ``source`` (where+ ``ConflictDetector`` reads it) and both carry ``metadata: {"source", "table"}``+ (``metadata`` is skipped by ``OntologyGenerator`` when it infers properties).++A junction relationship runs from the first foreign key listed to the second+(explicit keys come before schema-derived ones); its predicate is the second+key's, then the first's, then the table name.++Entity ids are ``"<Type>:<pk>"`` (``|`` and ``\\`` inside a key component are+backslash-escaped), so the same row ingested from two systems of record maps to+the same id and the two copies can be compared for conflicts.++Example:+ >>> mapper = RelationalSchemaMapper(+ ... entity_tables={+ ... "CUSTOMERS": {"pk": "CUSTOMER_ID", "type": "Customer", "name": "NAME"},+ ... "ORDERS": {"pk": "ORDER_ID", "type": "Order"},+ ... },+ ... foreign_keys=[+ ... {"table": "ORDERS", "column": "CUSTOMER_ID",+ ... "references": ("CUSTOMERS", "CUSTOMER_ID"), "predicate": "placedBy"},+ ... ],+ ... )+ >>> mapped = mapper.map({"CUSTOMERS": customers, "ORDERS": orders}, source="snowflake_crm")+ >>> GraphBuilder().build(sources=[mapped])+"""++from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Union++from ..utils.logging import get_logger++Row = Dict[str, Any]+TableInput = Union[Sequence[Row], Any]++# Keys the mapper sets itself, plus the aliases GraphBuilder promotes to+# "source" / "target" (a column named "object" would turn an entity into a+# relationship). A column with one of these names is rejected, not overwritten.+ENTITY_RESERVED = frozenset(+ {+ "id",+ "entity_id",+ "type",+ "name",+ "source",+ "source_id",+ "target",+ "target_id",+ "subject",+ "object",+ "metadata",+ }+)+RELATIONSHIP_RESERVED = frozenset(+ {+ "source",+ "source_id",+ "target",+ "target_id",+ "subject",+ "object",+ "type",+ "metadata",+ }+)+++class RelationalSchemaMapper:+ """Map relational rows plus a small schema spec to entities and relationships.++ Args:+ entity_tables: ``{table_name: {"pk": column or [columns], "type": ClassName,+ "name": column (optional)}}``. ``name`` names the column used as the+ entity's display name; it defaults to the primary key value.+ foreign_keys: Explicit foreign keys, ``[{"table", "column",+ "references": (table, column), "predicate" (optional)}]``. This is the+ required fallback for warehouses where constraints are informational+ or absent. ``predicate`` defaults to the referenced table's name. The+ referenced column must be that table's primary key (entity ids come+ from it), so a key into any other column is a ``ValueError``.+ schema: The dict returned by ``DBIngestor.get_database_schema()``; its+ ``foreign_keys`` entries (SQLAlchemy inspector dicts) are used when no+ explicit foreign key covers the same column. Constraints that point+ at anything but an entity table's whole primary key are skipped with+ a warning. The ingestor does not+ record which table a constraint belongs to, so a ``table_name`` /+ ``table`` key is used when present and the owning table is otherwise+ inferred from which mapped table has the column.+ """++ def __init__(+ self,+ entity_tables: Mapping[str, Mapping[str, Any]],+ foreign_keys: Optional[Iterable[Mapping[str, Any]]] = None,+ schema: Optional[Mapping[str, Any]] = None,+ ) -> None:+ self.logger = get_logger("schema_mapper")+ self.entity_tables: Dict[str, Dict[str, Any]] = {}+ for table, spec in entity_tables.items():+ if not spec.get("pk"):+ raise ValueError(f"entity table {table!r} needs a 'pk' column")+ if not spec.get("type"):+ raise ValueError(f"entity table {table!r} needs a 'type' class name")+ pk = spec["pk"]+ self.entity_tables[table] = {+ "pk": [pk] if isinstance(pk, str) else list(pk),+ "type": spec["type"],+ "name": spec.get("name"),+ }++ self._explicit: List[Dict[str, Any]] = [+ self._normalize_foreign_key(fk) for fk in (foreign_keys or [])+ ]+ self._schema_fks: List[Dict[str, Any]] = [+ fk for fk in self._foreign_keys_from_schema(schema) if fk is not None+ ]++ # ------------------------------------------------------------------ config++ def _normalize_foreign_key(self, fk: Mapping[str, Any]) -> Dict[str, Any]:+ table = fk.get("table")+ column = fk.get("column")+ references = fk.get("references")+ if not table or not column or not references or len(references) != 2:+ raise ValueError(+ "foreign key needs 'table', 'column' and 'references': (table, column); "+ f"got {dict(fk)!r}"+ )+ ref_table, ref_column = references+ if ref_table not in self.entity_tables:+ raise ValueError(+ f"foreign key on {table}.{column} references {ref_table!r}, "+ "which is not an entity table"+ )+ pk = self.entity_tables[ref_table]["pk"]+ if [ref_column] != pk:+ why = (+ "a composite primary key cannot be referenced"+ if len(pk) > 1+ else "a foreign key must reference the primary key"+ )+ raise ValueError(+ f"foreign key on {table}.{column} references {ref_table}.{ref_column}, "+ f"but {ref_table!r} entity ids come from {pk}; {why}"+ )+ return {+ "table": table,+ "column": column,+ "ref_table": ref_table,+ "ref_column": ref_column,+ "predicate": fk.get("predicate"), # None: defaults to the referenced table+ }++ def _foreign_keys_from_schema(+ self, schema: Optional[Mapping[str, Any]]+ ) -> Iterable[Optional[Dict[str, Any]]]:+ for fk in (schema or {}).get("foreign_keys", []) or []:+ columns = fk.get("constrained_columns") or []+ ref_columns = fk.get("referred_columns") or []+ ref_table = fk.get("referred_table")+ # composite constraints have no single entity id to point at+ if len(columns) != 1 or len(ref_columns) != 1 or not ref_table:+ yield None+ continue+ if ref_table not in self.entity_tables:+ yield None+ continue+ # a constraint on a unique non-key column has no entity id to point at+ pk = self.entity_tables[ref_table]["pk"]+ if list(ref_columns) != pk:+ self.logger.warning(+ "skipping schema foreign key %s.%s -> %s.%s: %r entity ids "+ "come from %s",+ fk.get("table_name") or fk.get("table") or "?",+ columns[0],+ ref_table,+ ref_columns[0],+ ref_table,+ pk,+ )+ yield None+ continue+ yield {+ "table": fk.get("table_name") or fk.get("table"),+ "column": columns[0],+ "ref_table": ref_table,+ "ref_column": ref_columns[0],+ "predicate": None,+ }++ def _foreign_keys_for(+ self, table: str, columns: Iterable[str]+ ) -> List[Dict[str, Any]]:+ """Foreign keys that apply to ``table``: explicit ones first, then schema+ ones for columns no explicit key already covers.++ Inference by column name is a heuristic: a table that merely copies a+ key column (a denormalised ``SHIPMENTS.CUSTOMER_ID``) is read as holding+ that foreign key. Pass ``foreign_keys`` explicitly when that is wrong.+ """+ column_set = set(columns)+ result = [fk for fk in self._explicit if fk["table"] == table]+ covered = {fk["column"] for fk in result}+ for fk in self._schema_fks:+ owner = fk["table"]+ if owner is not None and owner != table:+ continue+ if fk["column"] in covered:+ continue+ # a key recorded against this table is kept while the table is empty;+ # once there are columns, recorded and inferred keys both need theirs+ if fk["column"] not in column_set and (owner is None or column_set):+ continue+ # the referenced table carries that key column itself; without an+ # owning table recorded, do not read it as a self-reference.+ if (+ owner is None+ and table == fk["ref_table"]+ and fk["column"] == fk["ref_column"]+ ):+ continue+ result.append({**fk, "table": table})+ covered.add(fk["column"])+ return result++ # ------------------------------------------------------------------ mapping++ def map(+ self, tables: Mapping[str, TableInput], source: str+ ) -> Dict[str, List[Dict[str, Any]]]:+ """Map ``{table_name: rows}`` to entities and relationships.++ ``rows`` may be a list of dicts, an ingestor result exposing ``.data`` or+ ``.rows`` (Snowflake, Databricks, DBIngestor) or ``.dataframe``+ (PandasIngestor), one of the ``{"columns", "row_count", "rows"}``+ dicts in ``DBIngestor.ingest_database()["tables"]``, or a pandas+ DataFrame. Tables listed in+ ``entity_tables`` produce entities; other tables must be junction tables+ held together by exactly two foreign keys and produce relationships only.+ """+ entities: List[Dict[str, Any]] = []+ relationships: List[Dict[str, Any]] = []++ for table, raw in tables.items():+ rows = _rows_of(raw)+ columns = _columns_of(rows)+ foreign_keys = self._foreign_keys_for(table, columns)++ if table in self.entity_tables:+ for row in rows:+ entity = self._entity(table, row, source)+ if entity is None:+ continue+ entities.append(entity)+ relationships.extend(+ self._fk_relationships(+ table, row, entity["id"], foreign_keys, source+ )+ )+ elif len(foreign_keys) == 2:+ relationships.extend(+ self._junction_relationships(table, rows, foreign_keys, source)+ )+ else:+ raise ValueError(+ f"table {table!r} is not an entity table and has {len(foreign_keys)} "+ "foreign key(s); a junction table needs exactly two"+ )++ return {"entities": entities, "relationships": relationships}++ def _entity(self, table: str, row: Row, source: str) -> Optional[Dict[str, Any]]:+ spec = self.entity_tables[table]+ key_values = [row.get(column) for column in spec["pk"]]+ if any(_is_missing(value) for value in key_values):+ return None+ pk = _pk_text(key_values)+ name_column = spec["name"]+ name = row.get(name_column) if name_column else None+ entity: Dict[str, Any] = {+ "id": f"{spec['type']}:{pk}",+ "type": spec["type"],+ "name": str(name) if not _is_missing(name) else pk,+ }+ for column, value in row.items():+ if column in spec["pk"]:+ continue+ if column in ENTITY_RESERVED:+ if column == name_column:+ continue # already the entity's name+ raise ValueError(+ f"column {column!r} in table {table!r} clashes with a reserved "+ f"entity key; rename it (reserved: {sorted(ENTITY_RESERVED)})"+ )+ entity[column] = value+ entity["source"] = source+ entity["metadata"] = {"source": source, "table": table}+ return entity++ def _fk_relationships(+ self,+ table: str,+ row: Row,+ entity_id: str,+ foreign_keys: Sequence[Dict[str, Any]],+ source: str,+ ) -> Iterable[Dict[str, Any]]:+ for fk in foreign_keys:+ target = self._referenced_id(fk, row)+ if target is None:+ continue+ yield {+ "source": entity_id,+ "target": target,+ "type": fk["predicate"] or fk["ref_table"],+ "metadata": {"source": source, "table": table},+ }++ def _junction_relationships(+ self,+ table: str,+ rows: Sequence[Row],+ foreign_keys: Sequence[Dict[str, Any]],+ source: str,+ ) -> Iterable[Dict[str, Any]]:+ first, second = foreign_keys+ # an explicit predicate on either key names the relationship; the+ # second key's predicate wins because it names the target side.+ predicate = second["predicate"] or first["predicate"] or table+ key_columns = {first["column"], second["column"]}+ for row in rows:+ head = self._referenced_id(first, row)+ tail = self._referenced_id(second, row)+ if head is None or tail is None:+ continue+ relationship: Dict[str, Any] = {+ "source": head,+ "target": tail,+ "type": predicate,+ }+ for column, value in row.items():+ if column in key_columns:+ continue+ if column in RELATIONSHIP_RESERVED:+ raise ValueError(+ f"column {column!r} in junction table {table!r} clashes with a "+ f"reserved relationship key; rename it "+ f"(reserved: {sorted(RELATIONSHIP_RESERVED)})"+ )+ relationship[column] = value+ relationship["metadata"] = {"source": source, "table": table}+ yield relationship++ def _referenced_id(self, fk: Dict[str, Any], row: Row) -> Optional[str]:+ value = row.get(fk["column"])+ if _is_missing(value):+ return None+ return f"{self.entity_tables[fk['ref_table']]['type']}:{_pk_text([value])}"+++# ---------------------------------------------------------------------- helpers+++def _rows_of(raw: TableInput) -> List[Row]:+ """Accept row dicts, ingestor results, or a DataFrame."""+ if isinstance(raw, (list, tuple)):+ return [_without_missing(dict(row)) for row in raw]+ if isinstance(raw, Mapping) and isinstance(raw.get("rows"), (list, tuple)):+ return [_without_missing(dict(row)) for row in raw["rows"]]+ for attribute in ("data", "rows"):+ value = getattr(raw, attribute, None)+ if isinstance(value, (list, tuple)):+ return [_without_missing(dict(row)) for row in value]+ frame = getattr(raw, "dataframe", raw)+ if hasattr(frame, "to_dict"):+ return [_without_missing(row) for row in frame.to_dict("records")]+ raise TypeError(f"unsupported table input {type(raw).__name__}")+++def _without_missing(row: Row) -> Row:+ """pandas marks gaps as NaN / NaT / NA; the graph wants plain None."""+ return {key: (None if _is_missing(value) else value) for key, value in row.items()}+++def _is_missing(value: Any) -> bool:+ if value is None:+ return True+ try:+ import pandas as pd++ return bool(pd.isna(value))+ except (TypeError, ValueError):+ # lists and other non-scalars+ return False+++def _pk_text(values: Iterable[Any]) -> str:+ """Join key components so that ``|`` and ``\\`` inside a component cannot+ make two keys read as one; used for entity ids and relationship targets+ alike so the two always agree."""+ return "|".join(+ _key_text(value).replace("\\", "\\\\").replace("|", "\\|") for value in values+ )+++def _key_text(value: Any) -> str:+ """Key text that is the same whichever source the row came from.++ A pandas integer column with a gap is read back as float64, so a key of+ ``1`` arrives as ``1.0``; that must still map to ``"1"``.+ """+ if isinstance(value, float) and value.is_integer():+ return str(int(value))+ return str(value)+++def _columns_of(rows: Sequence[Row]) -> List[str]:+ seen: Dict[str, None] = {}+ for row in rows:+ for column in row:+ seen.setdefault(column, None)+ return list(seen)diff --git a/tests/kg/test_schema_mapper.py b/tests/kg/test_schema_mapper.pynew file mode 100644index 00000000..ae197036--- /dev/null+++ b/tests/kg/test_schema_mapper.py@@ -0,0 +1,705 @@+"""Tests for RelationalSchemaMapper (issue #1386, part 1)."""++import logging+import math+from types import SimpleNamespace++import pandas as pd+import pytest++from semantica.kg import RelationalSchemaMapper+from semantica.kg.schema_mapper import RelationalSchemaMapper as DirectImport++CUSTOMERS = [+ {"CUSTOMER_ID": 1, "NAME": "Acme", "ADDRESS": "1 Main St", "CREDIT_LIMIT": 1000},+ {"CUSTOMER_ID": 2, "NAME": "Globex", "ADDRESS": "2 Side St", "CREDIT_LIMIT": 500},+]+ORDERS = [+ {"ORDER_ID": 10, "CUSTOMER_ID": 1, "AMOUNT": 99.5},+ {"ORDER_ID": 11, "CUSTOMER_ID": 2, "AMOUNT": 12.0},+ {"ORDER_ID": 12, "CUSTOMER_ID": None, "AMOUNT": 1.0},+]+PRODUCTS = [+ {"PRODUCT_ID": "p1", "SKU": "SKU-1", "CATEGORY": "Electronics"},+ {"PRODUCT_ID": "p2", "SKU": "SKU-2", "CATEGORY": "Electronics"},+]+ORDER_ITEMS = [+ {"ORDER_ID": 10, "PRODUCT_ID": "p1", "QUANTITY": 2},+ {"ORDER_ID": 10, "PRODUCT_ID": "p2", "QUANTITY": 1},+ {"ORDER_ID": 11, "PRODUCT_ID": "p2", "QUANTITY": 3},+]++ENTITY_TABLES = {+ "CUSTOMERS": {"pk": "CUSTOMER_ID", "type": "Customer", "name": "NAME"},+ "ORDERS": {"pk": "ORDER_ID", "type": "Order"},+ "PRODUCTS": {"pk": "PRODUCT_ID", "type": "Product", "name": "SKU"},+}+FOREIGN_KEYS = [+ {+ "table": "ORDERS",+ "column": "CUSTOMER_ID",+ "references": ("CUSTOMERS", "CUSTOMER_ID"),+ "predicate": "placedBy",+ },+ {+ "table": "ORDER_ITEMS",+ "column": "ORDER_ID",+ "references": ("ORDERS", "ORDER_ID"),+ },+ {+ "table": "ORDER_ITEMS",+ "column": "PRODUCT_ID",+ "references": ("PRODUCTS", "PRODUCT_ID"),+ "predicate": "containsProduct",+ },+]+++def by_id(entities):+ return {entity["id"]: entity for entity in entities}+++def test_exported_from_semantica_kg():+ assert RelationalSchemaMapper is DirectImport+++def test_rows_become_entities_with_flat_properties_and_source():+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)++ mapped = mapper.map({"CUSTOMERS": CUSTOMERS}, source="snowflake_crm")++ assert set(mapped) == {"entities", "relationships"}+ assert mapped["relationships"] == []+ entities = by_id(mapped["entities"])+ assert set(entities) == {"Customer:1", "Customer:2"}+ acme = entities["Customer:1"]+ assert acme["type"] == "Customer"+ assert acme["name"] == "Acme"+ # non-key columns land flat on the entity so ConflictDetector can read them+ assert acme["ADDRESS"] == "1 Main St"+ assert acme["CREDIT_LIMIT"] == 1000+ assert "CUSTOMER_ID" not in acme+ assert acme["source"] == "snowflake_crm"+ assert acme["metadata"] == {"source": "snowflake_crm", "table": "CUSTOMERS"}+++def test_same_row_from_two_systems_shares_an_id():+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)+ crm = mapper.map({"CUSTOMERS": CUSTOMERS}, source="snowflake_crm")+ master = mapper.map(+ {"CUSTOMERS": [{"CUSTOMER_ID": 1, "NAME": "Acme", "ADDRESS": "1 Main Street"}]},+ source="databricks_master",+ )++ ids = {e["id"] for e in crm["entities"]} & {e["id"] for e in master["entities"]}+ assert ids == {"Customer:1"}+ sources = {e["source"] for e in crm["entities"] + master["entities"]}+ assert sources == {"snowflake_crm", "databricks_master"}+++def test_foreign_keys_become_typed_relationships():+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS+ )++ mapped = mapper.map({"CUSTOMERS": CUSTOMERS, "ORDERS": ORDERS}, source="crm")++ orders = by_id(mapped["entities"])+ assert orders["Order:10"]["AMOUNT"] == 99.5+ assert "CUSTOMER_ID" in orders["Order:10"], "FK column stays a property too"+ rels = mapped["relationships"]+ assert rels == [+ {+ "source": "Order:10",+ "target": "Customer:1",+ "type": "placedBy",+ "metadata": {"source": "crm", "table": "ORDERS"},+ },+ {+ "source": "Order:11",+ "target": "Customer:2",+ "type": "placedBy",+ "metadata": {"source": "crm", "table": "ORDERS"},+ },+ ]+ # a null FK produces no relationship and no error+++def test_junction_table_maps_to_relationships_only():+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS+ )++ mapped = mapper.map(+ {"ORDERS": ORDERS, "PRODUCTS": PRODUCTS, "ORDER_ITEMS": ORDER_ITEMS},+ source="crm",+ )++ assert not [+ e for e in mapped["entities"] if e["metadata"]["table"] == "ORDER_ITEMS"+ ]+ junction = [+ r for r in mapped["relationships"] if r["metadata"]["table"] == "ORDER_ITEMS"+ ]+ assert junction == [+ {+ "source": "Order:10",+ "target": "Product:p1",+ "type": "containsProduct",+ "QUANTITY": 2,+ "metadata": {"source": "crm", "table": "ORDER_ITEMS"},+ },+ {+ "source": "Order:10",+ "target": "Product:p2",+ "type": "containsProduct",+ "QUANTITY": 1,+ "metadata": {"source": "crm", "table": "ORDER_ITEMS"},+ },+ {+ "source": "Order:11",+ "target": "Product:p2",+ "type": "containsProduct",+ "QUANTITY": 3,+ "metadata": {"source": "crm", "table": "ORDER_ITEMS"},+ },+ ]+++def test_junction_predicate_defaults_to_table_name_when_not_given():+ fks = [+ {+ "table": "ORDER_ITEMS",+ "column": "ORDER_ID",+ "references": ("ORDERS", "ORDER_ID"),+ },+ {+ "table": "ORDER_ITEMS",+ "column": "PRODUCT_ID",+ "references": ("PRODUCTS", "PRODUCT_ID"),+ },+ ]+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, foreign_keys=fks)++ mapped = mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="crm")++ assert mapped["relationships"][0]["type"] == "ORDER_ITEMS"+++def test_foreign_keys_from_db_ingestor_schema():+ # The shape DBIngestor.get_database_schema()["foreign_keys"] returns+ # (SQLAlchemy inspector dicts). The ingestor does not record the owning+ # table, so it is inferred from which provided table has the column.+ schema = {+ "tables": ["CUSTOMERS", "ORDERS"],+ "foreign_keys": [+ {+ "name": "fk_orders_customer",+ "constrained_columns": ["CUSTOMER_ID"],+ "referred_table": "CUSTOMERS",+ "referred_columns": ["CUSTOMER_ID"],+ "referred_schema": None,+ }+ ],+ }+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)++ mapped = mapper.map({"CUSTOMERS": CUSTOMERS, "ORDERS": ORDERS}, source="pg")++ rels = mapped["relationships"]+ assert [(r["source"], r["target"], r["type"]) for r in rels] == [+ ("Order:10", "Customer:1", "CUSTOMERS"),+ ("Order:11", "Customer:2", "CUSTOMERS"),+ ]+++def test_schema_foreign_keys_use_table_name_when_present():+ schema = {+ "foreign_keys": [+ {+ "table_name": "ORDERS",+ "constrained_columns": ["CUSTOMER_ID"],+ "referred_table": "CUSTOMERS",+ "referred_columns": ["CUSTOMER_ID"],+ }+ ]+ }+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)+ mapped = mapper.map({"ORDERS": ORDERS[:1]}, source="pg")+ assert mapped["relationships"] == [+ {+ "source": "Order:10",+ "target": "Customer:1",+ "type": "CUSTOMERS",+ "metadata": {"source": "pg", "table": "ORDERS"},+ }+ ]+++def test_explicit_foreign_keys_override_schema_predicates():+ schema = {+ "foreign_keys": [+ {+ "table_name": "ORDERS",+ "constrained_columns": ["CUSTOMER_ID"],+ "referred_table": "CUSTOMERS",+ "referred_columns": ["CUSTOMER_ID"],+ }+ ]+ }+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS[:1], schema=schema+ )+ mapped = mapper.map({"ORDERS": ORDERS[:1]}, source="pg")+ assert [r["type"] for r in mapped["relationships"]] == ["placedBy"]+++ "wrap",+ [+ lambda rows: SimpleNamespace(data=rows), # SnowflakeData / DatabricksData+ lambda rows: SimpleNamespace(rows=rows), # TableData (DBIngestor)+ # DBIngestor.ingest_database()["tables"][name]+ lambda rows: {"columns": list(rows[0]), "row_count": len(rows), "rows": rows},+ lambda rows: SimpleNamespace(dataframe=pd.DataFrame(rows)), # PandasData+ lambda rows: pd.DataFrame(rows),+ ],+)+def test_accepts_ingestor_results_and_dataframes(wrap):+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)++ mapped = mapper.map({"CUSTOMERS": wrap(CUSTOMERS)}, source="x")++ assert [e["id"] for e in mapped["entities"]] == ["Customer:1", "Customer:2"]+ assert mapped["entities"][0]["ADDRESS"] == "1 Main St"+++def test_dataframe_gaps_become_none():+ frame = pd.DataFrame(+ [+ {+ "CUSTOMER_ID": 1,+ "CREDIT_LIMIT": 100,+ "SINCE": pd.Timestamp("2020-01-01"),+ },+ {"CUSTOMER_ID": 2, "CREDIT_LIMIT": None, "SINCE": None},+ ]+ )+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)++ entities = mapper.map({"CUSTOMERS": frame}, source="x")["entities"]++ assert entities[0]["CREDIT_LIMIT"] == 100+ assert entities[1]["CREDIT_LIMIT"] is None # float NaN column+ assert entities[1]["SINCE"] is None # NaT+ assert not any(isinstance(v, float) and math.isnan(v) for v in entities[1].values())+++def test_dataframe_integer_keys_with_gaps_keep_stable_ids():+ # an int column with a missing row is read back as float64 (1.0, NaN)+ frame = pd.DataFrame(+ [{"CUSTOMER_ID": 1, "NAME": "Acme"}, {"CUSTOMER_ID": None, "NAME": "x"}]+ )+ orders = pd.DataFrame(+ [{"ORDER_ID": 10, "CUSTOMER_ID": 1}, {"ORDER_ID": 11, "CUSTOMER_ID": None}]+ )+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS[:1]+ )++ mapped = mapper.map({"CUSTOMERS": frame, "ORDERS": orders}, source="df")++ assert [e["id"] for e in mapped["entities"]] == [+ "Customer:1",+ "Order:10",+ "Order:11",+ ]+ assert [(r["source"], r["target"]) for r in mapped["relationships"]] == [+ ("Order:10", "Customer:1")+ ]+ plain = mapper.map({"CUSTOMERS": CUSTOMERS[:1]}, source="rows")["entities"][0]["id"]+ assert plain == "Customer:1"+++def test_reserved_columns_are_rejected():+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS+ )+ with pytest.raises(ValueError, match="reserved entity key"):+ mapper.map({"CUSTOMERS": [{"CUSTOMER_ID": 1, "type": "gold"}]}, source="x")+ # GraphBuilder promotes "object"/"subject"/"target_id" to relationship ends+ with pytest.raises(ValueError, match="reserved entity key"):+ mapper.map({"CUSTOMERS": [{"CUSTOMER_ID": 1, "object": "x"}]}, source="x")+ with pytest.raises(ValueError, match="reserved relationship key"):+ mapper.map(+ {"ORDER_ITEMS": [{"ORDER_ID": 10, "PRODUCT_ID": "p1", "target": "z"}]},+ source="x",+ )+ # upper-case ERP columns do not clash+ mapped = mapper.map({"CUSTOMERS": [{"CUSTOMER_ID": 1, "TYPE": "gold"}]}, source="x")+ assert mapped["entities"][0]["TYPE"] == "gold"+ assert mapped["entities"][0]["type"] == "Customer"+++def test_junction_predicate_precedence():+ def fks(first_pred, second_pred):+ return [+ {+ "table": "ORDER_ITEMS",+ "column": "ORDER_ID",+ "references": ("ORDERS", "ORDER_ID"),+ **({"predicate": first_pred} if first_pred else {}),+ },+ {+ "table": "ORDER_ITEMS",+ "column": "PRODUCT_ID",+ "references": ("PRODUCTS", "PRODUCT_ID"),+ **({"predicate": second_pred} if second_pred else {}),+ },+ ]++ def predicate(first_pred, second_pred):+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=fks(first_pred, second_pred)+ )+ return mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")[+ "relationships"+ ][0]["type"]++ assert predicate("hasItem", "containsProduct") == "containsProduct"+ assert predicate("hasItem", None) == "hasItem"+ assert predicate(None, None) == "ORDER_ITEMS"+ # a predicate that happens to equal the referenced table name is still explicit+ assert predicate(None, "PRODUCTS") == "PRODUCTS"+++def test_junction_edge_cases():+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS+ )+ # a null key in a junction row is skipped+ rows = [{"ORDER_ID": 10, "PRODUCT_ID": None, "QUANTITY": 1}]+ assert mapper.map({"ORDER_ITEMS": rows}, source="x")["relationships"] == []++ # three foreign keys is not a junction table+ three = FOREIGN_KEYS + [+ {+ "table": "ORDER_ITEMS",+ "column": "CUSTOMER_ID",+ "references": ("CUSTOMERS", "CUSTOMER_ID"),+ }+ ]+ mapper3 = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, foreign_keys=three)+ with pytest.raises(ValueError, match="3 foreign key"):+ mapper3.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")++ # a junction table described only by the DB schema+ schema = {+ "foreign_keys": [+ {+ "table_name": "ORDER_ITEMS",+ "constrained_columns": ["ORDER_ID"],+ "referred_table": "ORDERS",+ "referred_columns": ["ORDER_ID"],+ },+ {+ "table_name": "ORDER_ITEMS",+ "constrained_columns": ["PRODUCT_ID"],+ "referred_table": "PRODUCTS",+ "referred_columns": ["PRODUCT_ID"],+ },+ ]+ }+ schema_mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)+ rel = schema_mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")[+ "relationships"+ ][0]+ assert (rel["source"], rel["target"], rel["type"]) == (+ "Order:10",+ "Product:p1",+ "ORDER_ITEMS",+ )+++def test_schema_inference_self_reference_and_limits():+ employees = {"EMPLOYEES": {"pk": "EMP_ID", "type": "Employee"}}+ schema = {+ "foreign_keys": [+ {+ "constrained_columns": ["MANAGER_ID"],+ "referred_table": "EMPLOYEES",+ "referred_columns": ["EMP_ID"],+ },+ # composite constraints have no single entity id: ignored+ {+ "constrained_columns": ["A", "B"],+ "referred_table": "EMPLOYEES",+ "referred_columns": ["EMP_ID", "X"],+ },+ ]+ }+ mapper = RelationalSchemaMapper(entity_tables=employees, schema=schema)+ rows = [{"EMP_ID": 1, "MANAGER_ID": None}, {"EMP_ID": 2, "MANAGER_ID": 1}]++ rels = mapper.map({"EMPLOYEES": rows}, source="hr")["relationships"]++ assert [(r["source"], r["target"], r["type"]) for r in rels] == [+ ("Employee:2", "Employee:1", "EMPLOYEES")+ ]++ # documented limit: inference by column name also matches a denormalised copy+ tables = {**ENTITY_TABLES, "SHIPMENTS": {"pk": "SHIPMENT_ID", "type": "Shipment"}}+ fk_schema = {+ "foreign_keys": [+ {+ "constrained_columns": ["CUSTOMER_ID"],+ "referred_table": "CUSTOMERS",+ "referred_columns": ["CUSTOMER_ID"],+ }+ ]+ }+ shipments = [{"SHIPMENT_ID": "s1", "CUSTOMER_ID": 1}]+ rels = RelationalSchemaMapper(entity_tables=tables, schema=fk_schema).map(+ {"SHIPMENTS": shipments}, source="x"+ )["relationships"]+ assert [(r["source"], r["target"]) for r in rels] == [("Shipment:s1", "Customer:1")]+++def test_composite_key_components_containing_the_separator_do_not_collide():+ mapper = RelationalSchemaMapper(+ entity_tables={"P": {"pk": ["A", "B"], "type": "Pair"}}+ )+ rows = [+ {"A": "x|y", "B": "z"},+ {"A": "x", "B": "y|z"},+ {"A": "x\\", "B": "y|z"},+ {"A": "x|y\\", "B": "z"},+ ]+ ids = [e["id"] for e in mapper.map({"P": rows}, source="x")["entities"]]+ assert len(set(ids)) == 4+++def test_relationship_targets_use_the_same_key_encoding_as_entity_ids():+ mapper = RelationalSchemaMapper(+ entity_tables={+ "C": {"pk": "CODE", "type": "C"},+ "O": {"pk": "ID", "type": "O"},+ },+ foreign_keys=[{"table": "O", "column": "CODE", "references": ("C", "CODE")}],+ )+ mapped = mapper.map(+ {"C": [{"CODE": "a\\|b"}], "O": [{"ID": 1, "CODE": "a\\|b"}]}, source="x"+ )+ ids = {e["id"] for e in mapped["entities"]}+ assert mapped["relationships"][0]["target"] in ids+++def test_metadata_is_skipped_by_the_ontology_property_generator():+ from semantica.ontology.property_generator import PropertyGenerator++ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)+ entity = mapper.map({"ORDERS": ORDERS[:1]}, source="crm")["entities"][0]++ properties = PropertyGenerator()._extract_data_properties([entity])++ assert "AMOUNT" in properties+ assert "metadata" not in properties+ assert "table" not in properties+++def test_junction_direction_follows_foreign_key_order():+ reversed_fks = [+ FOREIGN_KEYS[2],+ FOREIGN_KEYS[1],+ ] # PRODUCT_ID first, ORDER_ID second+ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=reversed_fks+ )+ rel = mapper.map({"ORDER_ITEMS": ORDER_ITEMS[:1]}, source="x")["relationships"][0]+ assert (rel["source"], rel["target"], rel["type"]) == (+ "Product:p1",+ "Order:10",+ "containsProduct",+ )+++def test_display_column_named_name_is_allowed():+ mapper = RelationalSchemaMapper(+ entity_tables={"customers": {"pk": "id", "type": "Customer", "name": "name"}}+ )+ entity = mapper.map(+ {"customers": [{"id": 1, "name": "Acme", "city": "SF"}]}, source="pg"+ )["entities"][0]+ assert entity == {+ "id": "Customer:1",+ "type": "Customer",+ "name": "Acme",+ "city": "SF",+ "source": "pg",+ "metadata": {"source": "pg", "table": "customers"},+ }+++def test_gaps_in_plain_rows_become_none_too():+ rows = [{"CUSTOMER_ID": 1, "ADDRESS": float("nan"), "SINCE": pd.NaT, "TAGS": ["a"]}]+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)+ entity = mapper.map({"CUSTOMERS": rows}, source="x")["entities"][0]+ assert entity["ADDRESS"] is None+ assert entity["SINCE"] is None+ assert entity["TAGS"] == ["a"]+++def test_composite_key_with_a_missing_component_is_skipped():+ mapper = RelationalSchemaMapper(+ entity_tables={"LINES": {"pk": ["ORDER_ID", "LINE_NO"], "type": "OrderLine"}}+ )+ rows = [{"ORDER_ID": 10, "LINE_NO": None}, {"ORDER_ID": 10, "LINE_NO": 2}]+ assert [e["id"] for e in mapper.map({"LINES": rows}, source="x")["entities"]] == [+ "OrderLine:10|2"+ ]+++def test_composite_primary_key_and_name_fallback():+ mapper = RelationalSchemaMapper(+ entity_tables={"LINES": {"pk": ["ORDER_ID", "LINE_NO"], "type": "OrderLine"}}+ )+ rows = [{"ORDER_ID": 10, "LINE_NO": 1, "QTY": 2}]++ entity = mapper.map({"LINES": rows}, source="x")["entities"][0]++ assert entity["id"] == "OrderLine:10|1"+ assert entity["name"] == "10|1"+ assert entity["QTY"] == 2+ assert "ORDER_ID" not in entity and "LINE_NO" not in entity+++def test_rows_without_a_primary_key_are_skipped():+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)+ rows = [{"CUSTOMER_ID": None, "NAME": "ghost"}, {"CUSTOMER_ID": 3, "NAME": "real"}]++ mapped = mapper.map({"CUSTOMERS": rows}, source="x")++ assert [e["id"] for e in mapped["entities"]] == ["Customer:3"]+++def test_unknown_table_without_foreign_keys_is_an_error():+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES)++ with pytest.raises(ValueError, match="MYSTERY"):+ mapper.map({"MYSTERY": [{"A": 1}]}, source="x")+++def test_config_validation():+ with pytest.raises(ValueError, match="pk"):+ RelationalSchemaMapper(entity_tables={"T": {"type": "Thing"}})+ with pytest.raises(ValueError, match="type"):+ RelationalSchemaMapper(entity_tables={"T": {"pk": "ID"}})+ with pytest.raises(ValueError, match="references"):+ RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES,+ foreign_keys=[{"table": "ORDERS", "column": "CUSTOMER_ID"}],+ )+ with pytest.raises(ValueError, match="entity table"):+ RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES,+ foreign_keys=[+ {+ "table": "ORDERS",+ "column": "CUSTOMER_ID",+ "references": ("NOPE", "ID"),+ }+ ],+ )+ # entity ids come from the primary key, so a key into any other column+ # would point at an id that no row produces+ with pytest.raises(ValueError, match="primary key"):+ RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES,+ foreign_keys=[+ {+ "table": "ORDERS",+ "column": "CUSTOMER_EMAIL",+ "references": ("CUSTOMERS", "EMAIL"),+ }+ ],+ )+ with pytest.raises(ValueError, match="composite"):+ RelationalSchemaMapper(+ entity_tables={"LINES": {"pk": ["ORDER_ID", "LINE_NO"], "type": "Line"}},+ foreign_keys=[+ {+ "table": "SHIPMENTS",+ "column": "ORDER_ID",+ "references": ("LINES", "ORDER_ID"),+ }+ ],+ )+++def test_schema_foreign_keys_to_non_primary_columns_are_skipped(caplog):+ schema = {+ "foreign_keys": [+ {+ "constrained_columns": ["CUSTOMER_EMAIL"],+ "referred_table": "CUSTOMERS",+ "referred_columns": ["EMAIL"],+ }+ ]+ }+ with caplog.at_level(logging.WARNING, logger="semantica.schema_mapper"):+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)+ orders = [{"ORDER_ID": 10, "CUSTOMER_EMAIL": "a@x"}]++ mapped = mapper.map({"ORDERS": orders}, source="pg")++ assert mapped["relationships"] == []+ assert any("CUSTOMERS.EMAIL" in r.getMessage() for r in caplog.records)+++def test_empty_junction_table_with_schema_foreign_keys_maps_to_nothing():+ schema = {+ "foreign_keys": [+ {+ "table_name": "ORDER_ITEMS",+ "constrained_columns": ["ORDER_ID"],+ "referred_table": "ORDERS",+ "referred_columns": ["ORDER_ID"],+ },+ {+ "table_name": "ORDER_ITEMS",+ "constrained_columns": ["PRODUCT_ID"],+ "referred_table": "PRODUCTS",+ "referred_columns": ["PRODUCT_ID"],+ },+ ]+ }+ mapper = RelationalSchemaMapper(entity_tables=ENTITY_TABLES, schema=schema)++ assert mapper.map({"ORDER_ITEMS": []}, source="pg") == {+ "entities": [],+ "relationships": [],+ }+ # rows that exist but lack a declared key column are still a mismatch+ with pytest.raises(ValueError, match="1 foreign key"):+ mapper.map({"ORDER_ITEMS": [{"ORDER_ID": 10, "QUANTITY": 1}]}, source="pg")+++def test_output_feeds_graph_builder():+ from semantica.kg import GraphBuilder++ mapper = RelationalSchemaMapper(+ entity_tables=ENTITY_TABLES, foreign_keys=FOREIGN_KEYS+ )+ mapped = mapper.map({"CUSTOMERS": CUSTOMERS, "ORDERS": ORDERS}, source="crm")++ graph = GraphBuilder().build(sources=[mapped])++ entities = by_id(graph["entities"])+ assert {"Customer:1", "Customer:2", "Order:10", "Order:11", "Order:12"} <= set(+ entities+ )+ assert entities["Customer:1"]["ADDRESS"] == "1 Main St"+ assert entities["Customer:1"]["source"] == "crm"+ edges = {(r["source"], r["target"], r["type"]) for r in graph["relationships"]}+ assert ("Order:10", "Customer:1", "placedBy") in edges+ assert not any(target.endswith(":None") for _, target, _ in edges)