Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

fix(export): wire every offered format to a runnable exporter (#1719)

Kevin72ea83140a09
diff --git a/semantica/cli.py b/semantica/cli.py
index 0e50b0c7..d36d8da1 100644
--- a/semantica/cli.py
+++ b/semantica/cli.py
@@ -4071,13 +4071,35 @@ def ontology_version(cli_ctx: CLIContext, local_json: bool) -> None:
# ─── Data Out ─────────────────────────────────────────────────────────────────
+# Formats this command can build out of a graph-store dump. Three are left out
+# because a dump of entities and relationships cannot feed them: OWL and SHACL
+# are serialized from an ontology (`semantica ontology shacl` generates the
+# shapes), and the distance matrix is computed from an Explorer session graph
+# (POST /api/export/distance-enriched). Offering them advertised a failure, and
+# offering OWL would advertise a document with no classes in it.
_EXPORT_FORMATS = [
"turtle", "jsonld", "ntriples", "rdfxml",
"parquet", "arrow", "csv", "json", "yaml",
- "graphml", "owl", "shacl", "arangodb", "distance-enriched",
+ "graphml", "arangodb",
]
+def _multi_file_destination_error(
+ format_name: str, compress: bool
+) -> click.ClickException:
+ """The error for asking a multi-file format for a single destination."""
+ if compress:
+ return click.ClickException(
+ f"--format {format_name} writes one file per collection, so "
+ f"--compress has no single file to compress; use --output and leave "
+ f"--compress off."
+ )
+ return click.ClickException(
+ f"--format {format_name} writes one file per collection, so stdout "
+ f"cannot carry it; pass --output to name where they go."
+ )
+
+
@main.command()
@click.option("--format", "fmt", type=click.Choice(_EXPORT_FORMATS),
default="json", show_default=True)
@@ -4094,13 +4116,19 @@ def export(
cli_ctx: CLIContext, fmt: str, output: Optional[str], with_provenance: bool,
filter_str: Optional[str], compress: bool, local_dry: bool, local_json: bool,
) -> None:
- """Export the graph in 14 supported formats.
+ """Export the graph in 11 supported formats.
+
+ \b
+ arrow, csv and parquet write one file per collection, so --output names a
+ base path there and these formats cannot go to stdout.
\b
Examples:
semantica export --format turtle --output graph.ttl
- semantica export --format parquet --with-provenance --output graph.parquet
+ semantica export --format parquet --output graph.parquet
+ writes graph_entities.parquet and graph_relationships.parquet
semantica export --format csv --filter "type:Person" --output persons.csv
+ writes persons_entities.csv and persons_relationships.csv
"""
cli_ctx = _require_ctx(cli_ctx)
@@ -4112,7 +4140,7 @@ def export(
try:
import tempfile
- from .export import get_export_method
+ from .export import MULTI_FILE_FORMATS, get_export_method
from .graph_store import get_nodes, get_relationships
from .graph_store.config import graph_store_config
@@ -4120,6 +4148,12 @@ def export(
if fn is None:
raise click.ClickException("Export method not available: export/knowledge_graph")
+ # Fail before the store is read. These routes write one file per
+ # collection, which is neither one stream nor one compressed file,
+ # and finding that out after a full export wastes the read.
+ if fmt in MULTI_FILE_FORMATS and (compress or not output):
+ raise _multi_file_destination_error(fmt, compress)
+
graph_db = dict(cli_ctx.config.to_dict().get("graph_db", {}))
backend = cli_ctx.store_backend or graph_db.pop("backend", None)
previous_graph_config = graph_store_config.get_all()
@@ -4153,30 +4187,50 @@ def export(
temp_handle.close()
target_output = temp_output
- fn(knowledge_graph, target_output, **kwargs)
+ written = fn(knowledge_graph, target_output, **kwargs)
except ImportError as exc:
raise click.ClickException(f"Export module not available: {exc}") from exc
- if compress:
- import gzip
- assert temp_output is not None
- compressed = gzip.compress(Path(temp_output).read_bytes())
- if output:
- Path(output).write_bytes(compressed)
- _ok(cli_ctx, f"Wrote compressed {output}")
- else:
- sys.stdout.buffer.write(compressed)
- elif output:
- _ok(cli_ctx, f"Wrote {output}")
- else:
- assert temp_output is not None
- try:
- click.echo(Path(temp_output).read_text(encoding="utf-8"))
- except UnicodeDecodeError:
- sys.stdout.buffer.write(Path(temp_output).read_bytes())
+ # A route that decides its own output names the files it wrote: Arrow
+ # and Parquet write one file per collection, so the path this command
+ # was given is a base name and is never created. Report what is there
+ # rather than the name that was asked for.
+ produced = [Path(p) for p in written] if written else []
+ try:
+ if produced and (compress or not output):
+ # A backstop: a route that starts writing several files is
+ # caught here too, not only the two rejected above.
+ raise _multi_file_destination_error(fmt, compress)
+ if compress:
+ import gzip
- if temp_output:
- Path(temp_output).unlink(missing_ok=True)
+ assert temp_output is not None
+ compressed = gzip.compress(Path(temp_output).read_bytes())
+ if output:
+ Path(output).write_bytes(compressed)
+ _ok(cli_ctx, f"Wrote compressed {output}")
+ else:
+ sys.stdout.buffer.write(compressed)
+ elif output:
+ if produced:
+ _ok(cli_ctx, f"Wrote {', '.join(str(p) for p in produced)}")
+ elif written is None:
+ _ok(cli_ctx, f"Wrote {output}")
+ else:
+ _warn(
+ cli_ctx,
+ "Nothing written: the store returned no entities or "
+ "relationships.",
+ )
+ else:
+ assert temp_output is not None
+ try:
+ click.echo(Path(temp_output).read_text(encoding="utf-8"))
+ except UnicodeDecodeError:
+ sys.stdout.buffer.write(Path(temp_output).read_bytes())
+ finally:
+ if temp_output:
+ Path(temp_output).unlink(missing_ok=True)
_run_with_error_handling(_action)
diff --git a/semantica/export/__init__.py b/semantica/export/__init__.py
index 77e9dd67..19fe8c21 100644
--- a/semantica/export/__init__.py
+++ b/semantica/export/__init__.py
@@ -191,6 +191,7 @@ from .graph_exporter import GraphExporter
from .json_exporter import JSONExporter
from .lpg_exporter import LPGExporter
from .methods import (
+ MULTI_FILE_FORMATS,
export_arango,
export_arrow,
export_csv,
@@ -265,6 +266,7 @@ __all__ = [
"export_arango",
"generate_report",
"get_export_method",
+ "MULTI_FILE_FORMATS",
"list_available_methods",
# Configuration
"ExportConfig",
diff --git a/semantica/export/arrow_exporter.py b/semantica/export/arrow_exporter.py
index 020c2d05..4a60da3e 100644
--- a/semantica/export/arrow_exporter.py
+++ b/semantica/export/arrow_exporter.py
@@ -213,7 +213,7 @@ class ArrowExporter:
file_path: Union[str, Path],
schema: Optional["pa.Schema"] = None,
**options,
- ) -> None:
+ ) -> List[Path]:
"""
Export data to Arrow IPC file(s).
@@ -228,6 +228,11 @@ class ArrowExporter:
schema: Arrow schema to use (default: auto-select based on data)
**options: Additional options
+ Returns:
+ The files written. A dictionary is written one file per key, so
+ this is how a caller learns which files a multi-file export
+ produced; a list is written to ``file_path``.
+
Raises:
ValidationError: If data type is unsupported
@@ -264,6 +269,17 @@ class ArrowExporter:
)
for key, value in data.items():
if isinstance(value, list):
+ if not value:
+ # An empty collection has nothing to write and
+ # _write_arrow refuses it, which aborts the call
+ # and leaves the remaining collections unwritten.
+ # export_knowledge_graph() below and
+ # ParquetExporter's dictionary branch skip it the
+ # same way.
+ self.logger.debug(
+ f"Skipping key '{key}': collection is empty"
+ )
+ continue
output_path = file_path.parent / f"{file_path.stem}_{key}.arrow"
# Select schema based on key
@@ -292,6 +308,7 @@ class ArrowExporter:
status="completed",
message=f"Exported {len(exported_files)} Arrow files",
)
+ return exported_files
elif isinstance(data, list):
# Single Arrow file
self.progress_tracker.update_tracking(
@@ -305,6 +322,7 @@ class ArrowExporter:
status="completed",
message=f"Exported Arrow to: {file_path}",
)
+ return [file_path]
else:
raise ValidationError(
f"Unsupported data type: {type(data)}. "
@@ -515,7 +533,7 @@ class ArrowExporter:
def export_knowledge_graph(
self, knowledge_graph: Dict[str, Any], base_path: Union[str, Path], **options
- ) -> None:
+ ) -> List[Path]:
"""
Export knowledge graph to multiple Arrow IPC files.
@@ -534,6 +552,11 @@ class ArrowExporter:
base_path: Base path for output files (without extension)
**options: Additional options passed to export methods
+ Returns:
+ The files written, in the order they were exported. An empty
+ collection is not written, so a graph with no relationships
+ produces one file rather than two.
+
Example:
>>> kg = {
... "entities": [...],
@@ -548,7 +571,7 @@ class ArrowExporter:
f"Exporting knowledge graph to Arrow files: base_path={base_path}"
)
- exported_files = []
+ exported_files: List[Path] = []
# Export entities
entities = knowledge_graph.get("entities", [])
@@ -582,6 +605,8 @@ class ArrowExporter:
else:
self.logger.warning("No data found in knowledge graph to export")
+ return exported_files
+
def _dict_to_struct(self, metadata: Dict[str, Any]) -> Dict[str, List]:
"""
Convert metadata dictionary to Arrow struct format.
diff --git a/semantica/export/csv_exporter.py b/semantica/export/csv_exporter.py
index 45f97d65..d0dba969 100644
--- a/semantica/export/csv_exporter.py
+++ b/semantica/export/csv_exporter.py
@@ -107,7 +107,7 @@ class CSVExporter:
fieldnames: Optional[List[str]] = None,
mode: str = "w",
**options,
- ) -> None:
+ ) -> List[Path]:
"""
Export data to CSV file(s).
@@ -126,6 +126,11 @@ class CSVExporter:
Raises:
ValidationError: If data type is unsupported
+ Returns:
+ The files written. A dictionary is written one file per key, so
+ this is how a caller learns which files a multi-file export
+ produced; a list is written to ``file_path``.
+
Example:
>>> # Single CSV file
>>> exporter.export([{"id": 1, "name": "A"}], "data.csv")
@@ -152,7 +157,7 @@ class CSVExporter:
# Handle different data structures
if isinstance(data, dict):
# Export each key as separate CSV file
- exported_files = []
+ exported_files: List[Path] = []
self.progress_tracker.update_tracking(
tracking_id, message=f"Exporting {len(data)} data groups..."
)
@@ -191,6 +196,7 @@ class CSVExporter:
status="completed",
message=f"Exported {len(exported_files)} CSV files",
)
+ return exported_files
elif isinstance(data, list):
# Single CSV file
self.progress_tracker.update_tracking(
@@ -205,6 +211,7 @@ class CSVExporter:
status="completed",
message=f"Exported CSV to: {file_path}",
)
+ return [file_path]
else:
raise ValidationError(
f"Unsupported data type: {type(data)}. "
@@ -377,7 +384,7 @@ class CSVExporter:
def export_knowledge_graph(
self, knowledge_graph: Dict[str, Any], base_path: Union[str, Path], **options
- ) -> None:
+ ) -> List[Path]:
"""
Export knowledge graph to multiple CSV files.
@@ -400,6 +407,11 @@ class CSVExporter:
base_path: Base path for output files (without extension)
**options: Additional options passed to export methods
+ Returns:
+ The files written, in the order they were exported. An empty
+ collection is not written, so a graph with no relationships
+ produces one file rather than two.
+
Example:
>>> kg = {
... "entities": [...],
@@ -466,6 +478,8 @@ class CSVExporter:
else:
self.logger.warning("No data found in knowledge graph to export")
+ return exported_files
+
def _write_csv(
self,
data: List[Dict[str, Any]],
diff --git a/semantica/export/endpoint_names.py b/semantica/export/endpoint_names.py
new file mode 100644
index 00000000..95c0b17b
--- /dev/null
+++ b/semantica/export/endpoint_names.py
@@ -0,0 +1,92 @@
+"""The names a relationship record may use for the two ends of an edge.
+
+The local graph store names them ``source_id`` and ``target_id``. The server
+backends name them ``start_node_id`` and ``end_node_id``: Neo4j, FalkorDB and
+Neptune directly, AGE after renaming its raw ``start_id``/``end_id`` edge
+fields. Every serializer reads only the ``source*`` and ``target*`` spellings,
+so a record written by a backend reads as having no endpoints at all.
+
+Anything that has to recognise a relationship, or hand one to a serializer,
+should take the names from here rather than restate them. Two copies of this
+list drift: the router recognised the server names while the Parquet classifier
+did not, and a store-shaped edge collection was written with the entity schema.
+"""
+
+from typing import Any, Dict, List, Tuple
+
+# Knowledge-graph keys whose values are relationship records.
+RELATIONSHIP_COLLECTIONS: Tuple[str, ...] = ("relationships", "edges")
+
+CANONICAL_SOURCE = "source_id"
+CANONICAL_TARGET = "target_id"
+
+# Accepted spellings, canonical first. Only names that cannot be an entity
+# field: `start` and `end` are the offset columns of every entity schema here
+# (`parquet_exporter.ENTITY_SCHEMA`, `arrow_exporter`, `csv_exporter`), and
+# reading them as endpoints turns an entity record into a relationship with
+# meaningless ids. `from`/`to`/`src`/`dst` are left to the relationship-only
+# normalizers in these exporters, which already know what they are holding.
+# ``export_relationships`` documents the accepted alternates as
+# ``source``/``source_id`` and ``target``/``target_id``.
+SOURCE_ENDPOINT_NAMES: Tuple[str, ...] = (
+ CANONICAL_SOURCE,
+ "source",
+ "start_node_id",
+ "start_id",
+)
+
+TARGET_ENDPOINT_NAMES: Tuple[str, ...] = (
+ CANONICAL_TARGET,
+ "target",
+ "end_node_id",
+ "end_id",
+)
+
+# Canonical name -> the other spellings accepted for it.
+ENDPOINT_ALIASES: Dict[str, Tuple[str, ...]] = {
+ CANONICAL_SOURCE: SOURCE_ENDPOINT_NAMES[1:],
+ CANONICAL_TARGET: TARGET_ENDPOINT_NAMES[1:],
+}
+
+
+def has_endpoints(sample: Dict[str, Any]) -> bool:
+ """Whether a record carries both ends of a relationship, under any name."""
+ return any(k in sample for k in SOURCE_ENDPOINT_NAMES) and any(
+ k in sample for k in TARGET_ENDPOINT_NAMES
+ )
+
+
+def canonical_endpoints(records: Any) -> Any:
+ """Return ``records`` with relationship endpoints under their canonical keys.
+
+ Records that already carry ``source_id``/``target_id`` are returned
+ untouched, as are non-dict records. The original list is returned when
+ nothing needed adding, so callers that pass a canonical graph pay nothing.
+ """
+ if not isinstance(records, list):
+ return records
+
+ normalized: List[Any] = []
+ changed = False
+ for record in records:
+ if not isinstance(record, dict):
+ normalized.append(record)
+ continue
+
+ resolved: Dict[str, Any] = {}
+ for canonical, aliases in ENDPOINT_ALIASES.items():
+ if record.get(canonical) is not None:
+ continue
+ for alias in aliases:
+ if record.get(alias) is not None:
+ resolved[canonical] = record[alias]
+ break
+
+ if resolved:
+ changed = True
+ merged = dict(record)
+ merged.update(resolved)
+ normalized.append(merged)
+ else:
+ normalized.append(record)
+ return normalized if changed else records
diff --git a/semantica/export/methods.py b/semantica/export/methods.py
index 33d129d4..74be36e7 100644
--- a/semantica/export/methods.py
+++ b/semantica/export/methods.py
@@ -160,7 +160,7 @@ Example Usage:
"""
from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
@@ -169,6 +169,7 @@ from .arango_aql_exporter import ArangoAQLExporter
from .arrow_exporter import ArrowExporter
from .config import export_config
from .csv_exporter import CSVExporter
+from .endpoint_names import RELATIONSHIP_COLLECTIONS, canonical_endpoints
from .graph_exporter import GraphExporter
from .json_exporter import JSONExporter
from .lpg_exporter import LPGExporter
@@ -297,7 +298,7 @@ def export_csv(
file_path: Union[str, Path],
method: str = "default",
**kwargs,
-) -> None:
+) -> Optional[List[Path]]:
"""
Export data to CSV format (convenience function).
@@ -309,6 +310,12 @@ def export_csv(
method: Export method (default: "default")
**kwargs: Additional options passed to CSVExporter
+ Returns:
+ The files written. A dictionary is written one file per key, so
+ ``file_path`` is a base name there and this is how the caller learns
+ what was actually produced. A registered custom method's own result is
+ returned in its place.
+
Examples:
>>> from semantica.export.methods import export_csv
>>> export_csv(entities, "entities.csv")
@@ -330,7 +337,7 @@ def export_csv(
config.update(kwargs)
exporter = CSVExporter(**config)
- exporter.export(data, file_path, **kwargs)
+ return exporter.export(data, file_path, **kwargs)
except Exception as e:
logger.error(f"Failed to export CSV: {e}")
@@ -342,7 +349,7 @@ def export_arrow(
file_path: Union[str, Path],
method: str = "default",
**kwargs,
-) -> None:
+) -> Optional[List[Path]]:
"""
Export data to Apache Arrow format (convenience function).
@@ -354,6 +361,12 @@ def export_arrow(
method: Export method (default: "default")
**kwargs: Additional options passed to ArrowExporter
+ Returns:
+ The files written. A dictionary is written one file per key, so
+ ``file_path`` is a base name there and this is how the caller learns
+ what was actually produced. A registered custom method's own result is
+ returned in its place.
+
Examples:
>>> from semantica.export.methods import export_arrow
>>> export_arrow(entities, "entities.arrow")
@@ -375,7 +388,7 @@ def export_arrow(
config.update(kwargs)
exporter = ArrowExporter(**config)
- exporter.export(data, file_path, **kwargs)
+ return exporter.export(data, file_path, **kwargs)
except Exception as e:
logger.error(f"Failed to export Arrow: {e}")
@@ -388,7 +401,7 @@ def export_parquet(
compression: str = "snappy",
method: str = "default",
**kwargs,
-) -> None:
+) -> Optional[List[Path]]:
"""
Export data to Apache Parquet format (convenience function).
@@ -408,6 +421,12 @@ def export_parquet(
method: Export method (default: "default")
**kwargs: Additional options passed to ParquetExporter
+ Returns:
+ The files written. A dictionary is written one file per key, so
+ ``file_path`` is a base name there and this is how the caller learns
+ what was actually produced. A registered custom method's own result is
+ returned in its place.
+
Examples:
>>> from semantica.export.methods import export_parquet
>>> export_parquet(entities, "entities.parquet", compression="snappy")
@@ -435,7 +454,7 @@ def export_parquet(
config.update(kwargs)
exporter = ParquetExporter(compression=compression, **config)
- exporter.export(data, file_path, **kwargs)
+ return exporter.export(data, file_path, **kwargs)
except Exception as e:
logger.error(f"Failed to export Parquet: {e}")
@@ -881,13 +900,52 @@ def generate_report(
raise
+# Collections whose records carry relationship endpoints, and the endpoint
+# names themselves, live in endpoint_names so the exporters that classify or
+# rewrite records share one table instead of keeping their own copy.
+
+
+# Formats whose route writes one file per collection instead of ``file_path``
+# and returns the files it wrote, so a caller can report or read the real output
+# rather than the base name it passed. A caller with room for one artifact
+# (stdout, a single compressed file) cannot take their output, so it is told
+# before the export runs rather than after. Kept beside the routing table it
+# describes; a test checks it against what the routes actually write.
+MULTI_FILE_FORMATS: Tuple[str, ...] = ("arrow", "csv", "parquet")
+
+
+def _one_collection_per_kind(
+ knowledge_graph: Union[Dict[str, Any], List[Dict[str, Any]]],
+) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+ """Give a dictionary one collection per kind, not one per spelling.
+
+ A knowledge graph arrives under the ``entities``/``relationships`` names or
+ the ``nodes``/``edges`` ones, and the command hands over both because the
+ exporters it routes to read different pairs. Arrow and Parquet write one
+ file per key, so an alias holding the same collection as its canonical name
+ is written a second time. An alias holding something else is left alone: it
+ is a second collection, not a second spelling of the first.
+ """
+ if not isinstance(knowledge_graph, dict):
+ return knowledge_graph
+
+ projected = dict(knowledge_graph)
+ for alias, canonical in (("nodes", "entities"), ("edges", "relationships")):
+ if alias not in projected or canonical not in projected:
+ continue
+ alias_value, canonical_value = projected[alias], projected[canonical]
+ if alias_value is canonical_value or alias_value == canonical_value:
+ del projected[alias]
+ return projected
+
+
def export_knowledge_graph(
knowledge_graph: Dict[str, Any],
file_path: Union[str, Path],
format: Optional[str] = None,
method: Optional[str] = None,
**kwargs,
-) -> None:
+) -> Optional[List[Path]]:
"""
Export knowledge graph to specified format (unified convenience function).
@@ -896,27 +954,41 @@ def export_knowledge_graph(
Args:
knowledge_graph: Knowledge graph dictionary with entities and
- relationships
+ relationships. Relationships may name their endpoints with any of
+ the spellings the graph stores use (``source_id``, ``source``,
+ ``start_node_id``, ``start_id``, ``from``, and the target-side
+ equivalents); they are normalized to ``source_id``/``target_id``
+ before routing, since that is all the serializers understand.
file_path: Output file path (format auto-detected from extension
if format not specified)
format: Export format (auto-detected from file extension if not
specified)
- "json", "json-ld": JSONExporter
- "csv": CSVExporter
+ - "parquet": ParquetExporter
+ - "arrow": ArrowExporter
- "ttl", "turtle": RDFExporter (turtle)
- "rdf", "rdfxml": RDFExporter (rdfxml)
- "graphml": GraphExporter (graphml)
- "gexf": GraphExporter (gexf)
- "dot": GraphExporter (dot)
- "yaml", "yml": YAML exporters
- - "owl": OWLExporter
+ - "owl", "owl-xml": OWLExporter, which takes an ontology dictionary
+ rather than a knowledge graph; a graph with no classes exports no
+ classes
- "cypher": LPGExporter
- - "aql": ArangoAQLExporter
+ - "aql", "arangodb": ArangoAQLExporter
method: Optional specific export method. Left as ``None``, each
exporter keeps its own default rather than being handed a value
the caller never chose.
**kwargs: Additional options passed to exporter
+ Returns:
+ The files an Arrow or Parquet route wrote, so a caller can report
+ or read them instead of the base name it passed; one file per
+ collection, which is not ``file_path``. The other routes write
+ ``file_path`` itself and return ``None``.
+
Examples:
>>> from semantica.export.methods import export_knowledge_graph
>>> export_knowledge_graph(kg, "output.json", format="json")
@@ -924,6 +996,18 @@ def export_knowledge_graph(
>>> export_knowledge_graph(kg, "output.ttl") # extension decides
>>> export_knowledge_graph(kg, "output.cypher", format="cypher")
"""
+ # Store-shaped relationship records carry their endpoints under names the
+ # serializers do not know. Normalize once here rather than in each exporter:
+ # this function is the single entry point every caller goes through, and the
+ # RDF serializers, the tabular exporters and the pass-through exporters
+ # (JSON, CSV, YAML) each read the records differently.
+ if isinstance(knowledge_graph, dict):
+ for key in RELATIONSHIP_COLLECTIONS:
+ collection = knowledge_graph.get(key)
+ canonical = canonical_endpoints(collection)
+ if canonical is not collection:
+ knowledge_graph = {**knowledge_graph, key: canonical}
+
# Auto-detect format from file extension if not specified
if not format:
file_path_obj = Path(file_path)
@@ -944,6 +1028,7 @@ def export_knowledge_graph(
".cypher": "cypher",
".aql": "aql",
".parquet": "parquet",
+ ".arrow": "arrow",
}
format = format_map.get(ext, "json")
@@ -953,11 +1038,21 @@ def export_knowledge_graph(
def _method_kwargs() -> Dict[str, Any]:
return {} if method is None else {"method": method}
+ # Arrow and Parquet write one file per collection rather than one file at
+ # ``file_path``, so a caller that has to report or consume the output needs
+ # the paths back instead of the base name it passed.
+ written: Optional[List[Path]] = None
+
# Route to appropriate exporter
if format in ["json", "json-ld"]:
export_json(knowledge_graph, file_path, format=format, **_method_kwargs(), **kwargs)
elif format == "csv":
- export_csv(knowledge_graph, file_path, **_method_kwargs(), **kwargs)
+ written = export_csv(
+ _one_collection_per_kind(knowledge_graph),
+ file_path,
+ **_method_kwargs(),
+ **kwargs,
+ )
elif format in ["turtle", "rdfxml", "jsonld", "ntriples", "n3"]:
export_rdf(knowledge_graph, file_path, format=format, **_method_kwargs(), **kwargs)
elif format in ["graphml", "gexf", "dot"]:
@@ -965,20 +1060,43 @@ def export_knowledge_graph(
elif format in ["yaml", "yml"]:
export_yaml(knowledge_graph, file_path, **_method_kwargs(), **kwargs)
elif format in ["owl-xml", "owl"]:
- export_owl(knowledge_graph, file_path, format=format, **_method_kwargs(), **kwargs)
+ # OWLExporter accepts "owl-xml" or "turtle". The extension map and the
+ # docs already call the default "owl-xml", so the short name is
+ # normalized the same way instead of being forwarded and rejected.
+ export_owl(
+ knowledge_graph,
+ file_path,
+ format="owl-xml" if format == "owl" else format,
+ **_method_kwargs(),
+ **kwargs,
+ )
elif format == "cypher":
export_lpg(knowledge_graph, file_path, **_method_kwargs(), **kwargs)
elif format in ["neo4j_csv", "neo4j-csv"]:
# file_path is treated as the output directory; nodes.csv and
# relationships.csv are written inside it.
export_neo4j_csv(knowledge_graph, file_path, **_method_kwargs(), **kwargs)
- elif format == "aql":
+ elif format in ["aql", "arangodb"]:
export_arango(knowledge_graph, file_path, **_method_kwargs(), **kwargs)
+ elif format == "arrow":
+ written = export_arrow(
+ _one_collection_per_kind(knowledge_graph),
+ file_path,
+ **_method_kwargs(),
+ **kwargs,
+ )
elif format == "parquet":
- export_parquet(knowledge_graph, file_path, **_method_kwargs(), **kwargs)
+ written = export_parquet(
+ _one_collection_per_kind(knowledge_graph),
+ file_path,
+ **_method_kwargs(),
+ **kwargs,
+ )
else:
raise ProcessingError(f"Unknown export format: {format}")
+ return written
+
def get_export_method(task: str, name: str) -> Optional[Callable]:
"""
diff --git a/semantica/export/parquet_exporter.py b/semantica/export/parquet_exporter.py
index fbe30a30..5ead4a20 100644
--- a/semantica/export/parquet_exporter.py
+++ b/semantica/export/parquet_exporter.py
@@ -88,6 +88,7 @@ from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.helpers import ensure_directory
from ..utils.logging import get_logger
from ..utils.progress_tracker import get_progress_tracker
+from .endpoint_names import canonical_endpoints, has_endpoints
class ParquetExporter:
@@ -160,13 +161,25 @@ class ParquetExporter:
self.logger.debug(f"Parquet exporter initialized: compression={compression}")
+ @staticmethod
+ def _records_look_like_relationships(sample: Dict[str, Any]) -> bool:
+ """Whether a record carries both endpoints of a relationship.
+
+ Used to pick a writer for records handed over without a key that names
+ them (a bare list, or a knowledge-graph key other than "entities" and
+ "relationships"). The endpoint names come from endpoint_names, so a
+ record from a server backend, which spells them ``start_node_id`` and
+ ``end_node_id``, is recognised the same as a local one.
+ """
+ return has_endpoints(sample)
+
def export(
self,
data: Union[List[Dict[str, Any]], Dict[str, Any]],
file_path: Union[str, Path],
schema: Optional["pa.Schema"] = None,
**options,
- ) -> None:
+ ) -> List[Path]:
"""
Export data to Parquet file(s).
@@ -184,6 +197,11 @@ class ParquetExporter:
Raises:
ValidationError: If data type is unsupported
+ Returns:
+ The files written. A dictionary is written one file per key, so
+ this is how a caller learns which files a multi-file export
+ produced; a list is written to ``file_path``.
+
Example:
>>> # Single Parquet file
>>> exporter.export([{"id": "1", "name": "A"}], "data.parquet")
@@ -210,7 +228,7 @@ class ParquetExporter:
# Handle different data structures
if isinstance(data, dict):
# Export each key as separate Parquet file
- exported_files = []
+ exported_files: List[Path] = []
self.progress_tracker.update_tracking(
tracking_id, message=f"Exporting {len(data)} data groups..."
)
@@ -231,17 +249,38 @@ class ParquetExporter:
file_path.parent / f"{file_path.stem}_{key}.parquet"
)
+ # A collection from a server backend names a
+ # relationship's ends start_node_id/end_node_id.
+ # Normalize before routing, so a store-shaped
+ # collection is both recognised and written under the
+ # canonical names. A caller who passed an explicit
+ # schema picked the columns already, so this cannot
+ # change what they asked for.
+ value = canonical_endpoints(value)
+
# Use dedicated export methods for entities and relationships
# to ensure proper normalization
- if key == "entities" and schema is None:
- self.export_entities(value, output_path, **options)
- elif key == "relationships" and schema is None:
- self.export_relationships(value, output_path, **options)
- else:
- # For other keys, write directly with provided schema
+ if schema is not None:
+ # An explicit schema is the caller saying what the
+ # records are, so it wins over the key name.
self._write_parquet(
value, output_path, schema=schema, **options
)
+ elif key == "entities":
+ self.export_entities(value, output_path, **options)
+ elif key == "relationships":
+ self.export_relationships(value, output_path, **options)
+ elif self._records_look_like_relationships(value[0]):
+ # A key that is neither. A caller may name a
+ # collection anything it likes, and the shape of
+ # the records is the only thing left to route on,
+ # the same way the list branch below has always
+ # done. Falling through to _write_parquet with
+ # schema=None would raise "Schema is required" no
+ # matter what the records hold.
+ self.export_relationships(value, output_path, **options)
+ else:
+ self.export_entities(value, output_path, **options)
exported_files.append(output_path)
else:
@@ -259,12 +298,18 @@ class ParquetExporter:
status="completed",
message=f"Exported {len(exported_files)} Parquet files",
)
+ return exported_files
elif isinstance(data, list):
# Single Parquet file - auto-detect if entities or relationships
self.progress_tracker.update_tracking(
tracking_id, message=f"Exporting {len(data)} records..."
)
+ # Same normalization as the dictionary branch: a bare list from
+ # a server backend has to be recognised and written under the
+ # canonical endpoint names.
+ data = canonical_endpoints(data)
+
# If no schema provided, try to auto-detect from data structure
if schema is None:
if not data:
@@ -274,14 +319,8 @@ class ParquetExporter:
"export_entities/export_relationships."
)
sample = data[0]
- has_source = any(
- k in sample for k in ["source_id", "source", "from_id", "from"]
- )
- has_target = any(
- k in sample for k in ["target_id", "target", "to_id", "to"]
- )
- if has_source and has_target:
+ if self._records_look_like_relationships(sample):
# Use dedicated method for relationship normalization
self.export_relationships(data, file_path, **options)
else:
@@ -297,6 +336,7 @@ class ParquetExporter:
status="completed",
message=f"Exported Parquet to: {file_path}",
)
+ return [file_path]
else:
raise ValidationError(
f"Unsupported data type: {type(data)}. "
@@ -571,7 +611,7 @@ class ParquetExporter:
def export_knowledge_graph(
self, kg: Dict[str, Any], base_path: Union[str, Path], **options
- ) -> None:
+ ) -> List[Path]:
"""
Export knowledge graph to multiple Parquet files.
@@ -584,6 +624,11 @@ class ParquetExporter:
base_path: Base path for output files (without extension)
**options: Additional options passed to export methods
+ Returns:
+ The files written, in the order they were exported. An empty
+ collection is not written, so a graph with no relationships
+ produces one file rather than two.
+
Raises:
ValidationError: If knowledge graph is missing required keys
@@ -645,6 +690,8 @@ class ParquetExporter:
message=f"Exported {len(exported_files)} Parquet files",
)
+ return exported_files
+
except Exception as e:
self.progress_tracker.stop_tracking(
tracking_id, status="failed", message=str(e)
diff --git a/tests/export/test_cli_export_formats.py b/tests/export/test_cli_export_formats.py
new file mode 100644
index 00000000..6cc22624
--- /dev/null
+++ b/tests/export/test_cli_export_formats.py
@@ -0,0 +1,520 @@
+"""Every format the ``semantica export`` command offers must produce its data.
+
+Issue #1712: the command advertised 14 formats and six of them could not run.
+Four had no branch at all in the router the command calls (``arrow``,
+``shacl``, ``arangodb``, ``distance-enriched``), ``owl`` was forwarded under a
+name ``OWLExporter`` rejects, and ``parquet`` aborted on the knowledge graph the
+command builds.
+
+OWL and SHACL are serialized from an ontology (``semantica ontology shacl``
+generates shapes) and the distance matrix is computed from an Explorer session
+graph. A graph-store dump has neither, so the command no longer offers the
+three formats rather than trading a loud failure for a document with no classes
+in it.
+
+Asserting "it did not raise" would have been too weak a check even for the
+formats that stayed. Every server graph backend returns relationship endpoints
+as ``start_node_id``/``end_node_id`` -- Neo4j, FalkorDB and Neptune directly,
+AGE after renaming its raw ``start_id``/``end_id`` fields -- while every
+serializer reads ``source_id``/``target_id``. Only the local store uses those
+names, so the command was broken for anyone running it against a server.
+Before the fix N-Triples dropped the relationship outright, Turtle and RDF/XML
+wrote an unresolved `#None` endpoint where one was expected, and Parquet
+refused the records. The assertions below therefore look for both node ids and
+the relationship type in what reaches disk.
+
+The existing CLI test did not catch any of this because its fake store returned
+``{"source": ..., "target": ...}``, which no backend returns.
+"""
+
+from pathlib import Path
+
+import pytest
+
+from semantica.cli import _EXPORT_FORMATS
+from semantica.export.endpoint_names import canonical_endpoints, has_endpoints
+from semantica.export.methods import MULTI_FILE_FORMATS, export_knowledge_graph
+from semantica.utils.exceptions import ProcessingError
+
+NODES = [
+ {"id": "n1", "type": "Person", "properties": {"name": "Alice"}},
+ {"id": "n2", "type": "Org", "properties": {"name": "Acme"}},
+]
+
+# What the server backends put in a relationship record (Neo4j, FalkorDB,
+# AGE and Neptune all return this shape).
+RELATIONSHIPS = [
+ {
+ "id": "r1",
+ "type": "KNOWS",
+ "start_node_id": "n1",
+ "end_node_id": "n2",
+ "properties": {"since": 1843},
+ }
+]
+
+# The extension each advertised format is normally written to, and whether the
+# file needs an Arrow reader to inspect it.
+FORMAT_EXTENSIONS = {
+ "turtle": ".ttl",
+ "jsonld": ".jsonld",
+ "ntriples": ".nt",
+ "rdfxml": ".rdf",
+ "parquet": ".parquet",
+ "arrow": ".arrow",
+ "csv": ".csv",
+ "json": ".json",
+ "yaml": ".yaml",
+ "graphml": ".graphml",
+ "arangodb": ".aql",
+}
+
+
+def cli_knowledge_graph(relationships=None):
+ """The dictionary ``semantica export`` hands to the router.
+
+ The command reads nodes and relationships out of the graph store and adds
+ ``nodes``/``edges`` alongside ``entities``/``relationships``, because the
+ graph exporters read one pair of names and the rest read the other.
+ """
+ rels = RELATIONSHIPS if relationships is None else relationships
+ return {
+ "entities": [dict(n) for n in NODES],
+ "relationships": [dict(r) for r in rels],
+ "nodes": [dict(n) for n in NODES],
+ "edges": [dict(r) for r in rels],
+ }
+
+
+def _files(tmp_path, stem):
+ return sorted(Path(tmp_path).glob(f"{stem}*"))
+
+
+def _blob(paths):
+ """Everything written as one string, so the assertions can be uniform.
+
+ Parquet and Arrow need an Arrow reader; the rest are written as text.
+ """
+ chunks = []
+ for path in paths:
+ if path.suffix in (".parquet", ".arrow"):
+ pa = pytest.importorskip("pyarrow", reason="reading the file needs pyarrow")
+ if path.suffix == ".parquet":
+ import pyarrow.parquet as pq
+
+ table = pq.read_table(path)
+ else:
+ import pyarrow.ipc as ipc
+
+ with pa.memory_map(str(path), "r") as source:
+ table = ipc.open_file(source).read_all()
+ chunks.append(f"{path.name} rows={table.num_rows} {table.to_pydict()}")
+ else:
+ chunks.append(path.read_text(encoding="utf-8", errors="replace"))
+ return "\n".join(chunks)
+
+
+class TestEveryAdvertisedFormatRuns:
+ """Drive the router the way the command does, for every offered format."""
+
+ @pytest.mark.parametrize("fmt", _EXPORT_FORMATS)
+ def test_format_writes_its_graph(self, tmp_path, fmt):
+ stem = f"export_{fmt}"
+ target = tmp_path / f"{stem}{FORMAT_EXTENSIONS[fmt]}"
+
+ export_knowledge_graph(cli_knowledge_graph(), str(target), format=fmt)
+
+ files = _files(tmp_path, stem)
+ assert files, f"{fmt} wrote no file"
+ assert all(f.stat().st_size for f in files), f"{fmt} wrote an empty file"
+
+ blob = _blob(files)
+ assert "KNOWS" in blob, f"{fmt} lost the relationship type"
+ assert "n1" in blob and "n2" in blob, f"{fmt} lost a relationship endpoint"
+ # `#None` is the IRI fragment the RDF serializers fell back to when
+ # they could not resolve an endpoint before the store's names were
+ # understood: `<...#None>` in Turtle, `rdf:about="...#None"` in RDF/XML.
+ assert "#None" not in blob, f"{fmt} wrote an unresolved endpoint"
+
+ @pytest.mark.parametrize("fmt", _EXPORT_FORMATS)
+ def test_this_module_covers_exactly_what_the_command_offers(self, fmt):
+ """A format added to the command has to be covered here in the same change."""
+ assert (
+ fmt in FORMAT_EXTENSIONS
+ ), f"{fmt} is advertised but this module does not build it"
+
+
+class TestGraphStoreEndpointNames:
+ """The store's endpoint names are accepted on the way in.
+
+ The backend's spelling is not the one the serializers read, so a record
+ has to keep its endpoints through the router whichever name it arrived
+ under.
+ """
+
+ @pytest.mark.parametrize(
+ "source_key,target_key",
+ [
+ ("start_node_id", "end_node_id"), # Neo4j, FalkorDB, AGE, Neptune
+ ("start_id", "end_id"), # AGE's raw edge fields
+ ("source", "target"),
+ ("source_id", "target_id"), # already canonical
+ ],
+ )
+ def test_endpoints_survive_regardless_of_their_name(
+ self, tmp_path, source_key, target_key
+ ):
+ relationships = [
+ {
+ "id": "r1",
+ "type": "KNOWS",
+ source_key: "n1",
+ target_key: "n2",
+ }
+ ]
+ target = tmp_path / f"{source_key}.ttl"
+
+ export_knowledge_graph(
+ cli_knowledge_graph(relationships), str(target), format="turtle"
+ )
+
+ body = target.read_text(encoding="utf-8")
+ assert (
+ "n1" in body and "n2" in body
+ ), f"endpoints named {source_key}/{target_key} did not reach the output"
+ assert "#None" not in body
+
+ def test_relationships_reach_parquet_with_the_store_names(self, tmp_path):
+ """Parquet used to refuse these records outright.
+
+ ``export_relationships`` only recognised ``source*``/``target*``, so a
+ store-shaped record produced no valid rows and raised.
+ """
+ pytest.importorskip("pyarrow", reason="the parquet writer needs pyarrow")
+
+ export_knowledge_graph(cli_knowledge_graph(), str(tmp_path / "o.parquet"))
+
+ blob = _blob(_files(tmp_path, "o_relationships"))
+ assert "n1" in blob and "n2" in blob
+
+
+class TestParquetAcceptsTheCommandShape:
+ """``parquet`` is given the whole knowledge graph in one dict.
+
+ ``entities``/``relationships`` are handled by key name, and the duplicate
+ ``nodes``/``edges`` keys the command adds used to reach
+ ``_write_parquet(schema=None)``, which always raised
+ ``Schema is required for Parquet export``. They name the same two
+ collections, so the router drops them before routing instead of writing
+ each collection a second time.
+ """
+
+ def test_duplicate_collection_keys_are_not_written_as_well(self, tmp_path):
+ pytest.importorskip("pyarrow", reason="the parquet writer needs pyarrow")
+
+ export_knowledge_graph(cli_knowledge_graph(), str(tmp_path / "graph.parquet"))
+
+ names = sorted(p.name for p in tmp_path.glob("graph*"))
+ assert names == [
+ "graph_entities.parquet",
+ "graph_relationships.parquet",
+ ], names
+
+ def test_a_key_that_is_neither_entities_nor_relationships_is_routed_by_shape(
+ self, tmp_path
+ ):
+ pytest.importorskip("pyarrow", reason="the parquet writer needs pyarrow")
+
+ kg = {
+ "things": [{"id": "n1", "source_id": "a", "target_id": "b", "type": "REL"}]
+ }
+ export_knowledge_graph(kg, str(tmp_path / "shaped.parquet"))
+
+ blob = _blob(_files(tmp_path, "shaped_things"))
+ assert "a" in blob and "b" in blob
+
+ def test_an_explicit_schema_still_wins_over_the_key_name(self, tmp_path):
+ """The documented way to write an arbitrary key is to pass a schema.
+
+ A dictionary is still written one file per key, so the file to read is
+ the one named after the key rather than the path that was asked for.
+ """
+ pyarrow = pytest.importorskip(
+ "pyarrow", reason="the parquet writer needs pyarrow"
+ )
+ import pyarrow.parquet as pq
+
+ schema = pyarrow.schema(
+ [
+ pyarrow.field("id", pyarrow.string()),
+ pyarrow.field("note", pyarrow.string()),
+ ]
+ )
+ kg = {"entities": [{"id": "n1", "note": "kept"}]}
+
+ export_knowledge_graph(kg, str(tmp_path / "with_schema.parquet"), schema=schema)
+
+ # Had the key name won, this would carry the entity schema's columns.
+ target = tmp_path / "with_schema_entities.parquet"
+ assert target.exists(), sorted(p.name for p in tmp_path.glob("*"))
+ table = pq.read_table(target)
+ assert table.column_names == ["id", "note"]
+ assert table.to_pydict()["note"] == ["kept"]
+
+
+class TestOnlyRunnableFormatsAreOffered:
+ """The command's list is what this module covers."""
+
+ @pytest.mark.parametrize("fmt", ["owl", "shacl", "distance-enriched"])
+ def test_ontology_and_session_formats_are_not_offered(self, fmt):
+ """Three formats need an input a graph-store dump does not have.
+
+ OWL and SHACL are serialized from an ontology (``semantica ontology
+ shacl`` generates shapes; ``OWLExporter`` takes ``classes`` and
+ ``properties``), and the distance matrix is computed from an Explorer
+ session graph. Offering them here only advertised failures, so they
+ were dropped rather than faked.
+ """
+ assert fmt not in _EXPORT_FORMATS
+
+ @pytest.mark.parametrize("fmt", ["shacl", "distance-enriched"])
+ def test_the_router_does_not_claim_to_build_them(self, fmt):
+ with pytest.raises(ProcessingError, match="Unknown export format"):
+ export_knowledge_graph(cli_knowledge_graph(), "unused.out", format=fmt)
+
+ def test_owl_is_routed_but_produces_no_classes_for_a_graph(self, tmp_path):
+ """``owl`` is still routable; the short name is normalized.
+
+ ``OWLExporter`` accepts ``owl-xml`` or ``turtle`` and rejected the bare
+ ``owl`` the router forwarded, while the extension map had always mapped
+ ``.owl`` to ``owl-xml``. A knowledge graph carries no classes, so the
+ document is empty of them -- which is why the command does not offer
+ the format.
+ """
+ export_knowledge_graph(
+ cli_knowledge_graph(), str(tmp_path / "o.owl"), format="owl"
+ )
+
+ body = (tmp_path / "o.owl").read_text(encoding="utf-8")
+ assert body.lstrip().startswith("<")
+ assert "KNOWS" not in body
+
+
+class TestExtensionRoutingMatchesTheExplicitFormat:
+ """Naming the format and letting the extension name it must agree.
+
+ ``format_map`` carried ``.parquet`` but not ``.arrow``, and an extension it
+ does not know falls back to ``json``. Routing the Arrow branch by hand
+ therefore left callers of the documented extension-driven form with JSON
+ under an ``.arrow`` name, and no error. Comparing the files each spelling
+ writes catches that without depending on how a format looks inside.
+ """
+
+ @pytest.mark.parametrize("fmt", _EXPORT_FORMATS)
+ def test_the_extension_alone_reaches_the_same_exporter(self, tmp_path, fmt):
+ ext = FORMAT_EXTENSIONS[fmt]
+ explicit_dir = tmp_path / "explicit"
+ inferred_dir = tmp_path / "inferred"
+ explicit_dir.mkdir()
+ inferred_dir.mkdir()
+
+ export_knowledge_graph(
+ cli_knowledge_graph(), str(explicit_dir / f"graph{ext}"), format=fmt
+ )
+ export_knowledge_graph(cli_knowledge_graph(), str(inferred_dir / f"graph{ext}"))
+
+ written = {
+ name: sorted(p.name for p in directory.iterdir())
+ for name, directory in (
+ ("explicit", explicit_dir),
+ ("inferred", inferred_dir),
+ )
+ }
+ assert (
+ written["inferred"] == written["explicit"]
+ ), f"{ext} alone routed away from the {fmt} exporter: {written}"
+
+ files = [inferred_dir / name for name in written["inferred"]]
+ blob = _blob(files)
+ assert "KNOWS" in blob, f"{ext} alone lost the relationship type"
+ assert "n1" in blob and "n2" in blob, f"{ext} alone lost an endpoint"
+ assert "#None" not in blob, f"{ext} alone left an endpoint unresolved"
+
+
+class TestParquetRecognisesStoreShapedEdges:
+ """The shape check behind Parquet's routing reads the same name table.
+
+ It listed only the ``source*``/``target*`` spellings, so a caller passing a
+ store-shaped edge collection under a key of their own choosing had it
+ written with the entity schema, which succeeds while dropping both
+ endpoints. ``export_knowledge_graph`` normalizes first, so only a direct
+ call to the exporter could see this.
+ """
+
+ def test_a_store_shaped_edges_key_is_written_as_relationships(self, tmp_path):
+ pytest.importorskip("pyarrow", reason="the parquet writer needs pyarrow")
+ from semantica.export import ParquetExporter
+
+ ParquetExporter().export(
+ {
+ "edges": [
+ {
+ "id": "r1",
+ "type": "KNOWS",
+ "start_node_id": "n1",
+ "end_node_id": "n2",
+ }
+ ]
+ },
+ str(tmp_path / "direct.parquet"),
+ )
+
+ blob = _blob(_files(tmp_path, "direct_edges"))
+ assert "KNOWS" in blob
+ # The entity writer keeps id and type and drops the endpoints, so this
+ # is what tells the two writers apart.
+ assert "n1" in blob and "n2" in blob, "the endpoints did not reach the file"
+
+ def test_a_bare_list_of_store_shaped_edges_is_written_as_relationships(
+ self, tmp_path
+ ):
+ pytest.importorskip("pyarrow", reason="the parquet writer needs pyarrow")
+ from semantica.export import ParquetExporter
+
+ ParquetExporter().export(
+ [
+ {
+ "id": "r1",
+ "type": "KNOWS",
+ "start_node_id": "n1",
+ "end_node_id": "n2",
+ }
+ ],
+ str(tmp_path / "bare.parquet"),
+ )
+
+ blob = _blob(_files(tmp_path, "bare.parquet"))
+ assert "n1" in blob and "n2" in blob, "the endpoints did not reach the file"
+
+
+class TestEntityOffsetsAreNotEndpoints:
+ """``start`` and ``end`` are entity offsets, not the ends of a relationship.
+
+ Both are columns of the entity schema that ``parquet_exporter``,
+ ``arrow_exporter`` and ``csv_exporter`` write. A table of endpoint names
+ that includes them adds ``source_id``/``target_id`` to every entity record,
+ and the shape check behind Parquet's routing then reads that as a
+ relationship and hands the entities to the relationship writer, which drops
+ them and raises. The names in ``endpoint_names`` are therefore limited to
+ ones an entity field cannot be.
+ """
+
+ def test_a_record_with_offsets_is_not_read_as_a_relationship(self):
+ entity = {"id": "e1", "type": "Person", "start": 0, "end": 5}
+
+ assert has_endpoints(entity) is False
+ assert canonical_endpoints([entity]) == [entity]
+
+ def test_the_router_adds_no_endpoint_keys_to_entities(self, tmp_path):
+ """A writer that copies records through shows the damage directly.
+
+ The tabular writers pick named fields, so an extra key on an entity is
+ invisible there; JSON writes the record out as it stands.
+ """
+ kg = {"entities": [{"id": "e1", "type": "Person", "start": 0, "end": 5}]}
+ target = tmp_path / "entities.json"
+
+ export_knowledge_graph(kg, str(target), format="json")
+
+ body = target.read_text(encoding="utf-8")
+ assert "source_id" not in body and "target_id" not in body
+
+ def test_entity_offsets_survive_the_export(self, tmp_path):
+ pytest.importorskip("pyarrow", reason="the parquet writer needs pyarrow")
+ import pyarrow.parquet as pq
+
+ kg = {
+ "entities": [
+ {"id": "e1", "type": "Person", "start": 0, "end": 5},
+ {"id": "e2", "type": "Person", "start": 6, "end": 9},
+ ]
+ }
+ export_knowledge_graph(kg, str(tmp_path / "offsets.parquet"))
+
+ rows = pq.read_table(tmp_path / "offsets_entities.parquet").to_pydict()
+ assert rows["id"] == ["e1", "e2"]
+ assert rows["start"] == [0, 6]
+ assert rows["end"] == [5, 9]
+
+
+class TestMultiFileFormats:
+ """The formats that write one file per collection.
+
+ ``arrow``, ``csv`` and ``parquet`` treat the path they are handed as a base
+ name: ``graph.arrow`` becomes ``graph_entities.arrow`` and
+ ``graph_relationships.arrow``, and ``graph.arrow`` itself is never created.
+ A caller therefore has to be told what was written rather than what it
+ asked for, and cannot send one of these formats to a single destination.
+ """
+
+ @pytest.mark.parametrize("fmt", MULTI_FILE_FORMATS)
+ def test_writes_one_file_per_collection(self, tmp_path, fmt):
+ stem = f"base_{fmt}"
+ ext = FORMAT_EXTENSIONS[fmt]
+ target = tmp_path / f"{stem}{ext}"
+
+ export_knowledge_graph(cli_knowledge_graph(), str(target), format=fmt)
+
+ names = sorted(p.name for p in tmp_path.glob(f"{stem}*"))
+ assert names == [f"{stem}_entities{ext}", f"{stem}_relationships{ext}"], names
+ assert (
+ not target.exists()
+ ), "the path passed in is a base name, not the artifact"
+
+ @pytest.mark.parametrize("fmt", _EXPORT_FORMATS)
+ def test_returns_the_paths_it_wrote(self, tmp_path, fmt):
+ target = tmp_path / f"ret{FORMAT_EXTENSIONS[fmt]}"
+
+ written = export_knowledge_graph(cli_knowledge_graph(), str(target), format=fmt)
+
+ if fmt in MULTI_FILE_FORMATS:
+ assert written, f"{fmt} wrote files but reported none"
+ assert sorted(Path(p).name for p in written) == sorted(
+ p.name for p in tmp_path.glob("ret*")
+ )
+ else:
+ assert written is None, f"{fmt} writes the path it was given"
+
+ @pytest.mark.parametrize("fmt", _EXPORT_FORMATS)
+ def test_several_files_exactly_when_it_is_listed(self, tmp_path, fmt):
+ """``MULTI_FILE_FORMATS`` makes the command refuse a single destination,
+ so it has to agree with what the routes actually do."""
+ target = tmp_path / f"multi{FORMAT_EXTENSIONS[fmt]}"
+
+ export_knowledge_graph(cli_knowledge_graph(), str(target), format=fmt)
+
+ several = len(list(tmp_path.glob("multi*"))) > 1
+ assert several == (
+ fmt in MULTI_FILE_FORMATS
+ ), f"{fmt} wrote {several} files, listed={fmt in MULTI_FILE_FORMATS}"
+
+ @pytest.mark.parametrize("fmt", _EXPORT_FORMATS)
+ def test_a_graph_without_relationships_still_exports(self, tmp_path, fmt):
+ """Nothing to relate is not a failure (#1712).
+
+ Arrow refused an empty collection outright, so a graph whose nodes had
+ no edges between them could not be exported at all.
+ """
+ target = tmp_path / f"lonely{FORMAT_EXTENSIONS[fmt]}"
+
+ export_knowledge_graph(
+ cli_knowledge_graph(relationships=[]), str(target), format=fmt
+ )
+
+ files = _files(tmp_path, "lonely")
+ assert files, f"{fmt} wrote no file"
+ assert all(f.stat().st_size for f in files), f"{fmt} wrote an empty file"
+ # The entity ids, since the entity name lives in a nested property that
+ # the serializers do not all flatten.
+ blob = _blob(files)
+ assert "n1" in blob and "n2" in blob, f"{fmt} lost the entities"
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py
index 67114925..947baa84 100644
--- a/tests/test_cli_commands.py
+++ b/tests/test_cli_commands.py
@@ -1881,12 +1881,80 @@ class TestOntology:
# ─── export ───────────────────────────────────────────────────────────────────
+# The record shape the Neo4j and FalkorDB backends return: endpoints named
+# start_node_id/end_node_id, and get_nodes() nesting its fields under
+# "properties". The local store uses source_id/target_id, so a fixture written
+# against it hid the fact that the command fed the exporters a shape they could
+# not read (#1712).
+STORE_NODES = [
+ {"id": "n1", "type": "Person", "name": "Alice", "properties": {"name": "Alice"}}
+]
+STORE_RELATIONSHIP = {
+ "id": "r1",
+ "start_node_id": "n1",
+ "end_node_id": "n1",
+ "type": "KNOWS",
+ "properties": {},
+}
+
+# The extension each multi-file format is written under, checked against the
+# list the command refuses a single destination for.
+MULTI_FILE_EXTENSIONS = {"arrow": ".arrow", "csv": ".csv", "parquet": ".parquet"}
+
+
+def _patch_graph_store(monkeypatch, relationships, nodes=None) -> dict:
+ """Point the export command at a fake store, and count how often it reads."""
+ node_records = STORE_NODES if nodes is None else nodes
+ reads = {"nodes": 0}
+
+ def get_nodes(**kwargs):
+ reads["nodes"] += 1
+ return [dict(n) for n in node_records]
+
+ def get_relationships(**kwargs):
+ return [dict(r) for r in relationships]
+
+ class FakeGraphStore:
+ def get_nodes(self, **kwargs):
+ return get_nodes(**kwargs)
+
+ def get_relationships(self, **kwargs):
+ return get_relationships(**kwargs)
+
+ monkeypatch.setattr(
+ "semantica.graph_store.methods._get_store", lambda: FakeGraphStore()
+ )
+ for target in (
+ "semantica.graph_store.get_nodes",
+ "semantica.graph_store.methods.get_nodes",
+ ):
+ monkeypatch.setattr(target, get_nodes)
+ for target in (
+ "semantica.graph_store.get_relationships",
+ "semantica.graph_store.methods.get_relationships",
+ ):
+ monkeypatch.setattr(target, get_relationships)
+ return reads
+
+
+def test_multi_file_extensions_cover_what_the_command_lists():
+ from semantica.export import MULTI_FILE_FORMATS
+
+ assert set(MULTI_FILE_EXTENSIONS) == set(
+ MULTI_FILE_FORMATS
+ ), "a multi-file format was added or removed without updating these tests"
+
+
class TestExport:
- def test_help_shows_14_formats(self, runner):
+ def test_help_shows_the_offered_formats(self, runner):
result = runner.invoke(cli_module.main, ["export", "--help"])
_ok(result)
- for fmt in ["turtle", "parquet", "csv", "graphml", "owl", "arangodb"]:
+ for fmt in cli_module._EXPORT_FORMATS:
assert fmt in result.output
+ # These three need an ontology or an Explorer session, not a graph
+ # dump, so they are no longer offered (#1712).
+ for fmt in ["owl", "shacl", "distance-enriched"]:
+ assert fmt not in result.output
for flag in ["--with-provenance", "--filter", "--compress", "--dry-run"]:
assert flag in result.output
@@ -1908,78 +1976,7 @@ class TestExport:
assert data["dry_run"] is True
def test_real_export_runtime_path(self, runner, tmp_path, monkeypatch):
- class FakeGraphStore:
- def get_nodes(self, labels=None, properties=None, limit=100, **options):
- return [
- {
- "id": "n1",
- "type": "Person",
- "name": "Alice",
- "properties": {"name": "Alice"},
- }
- ]
-
- def get_relationships(self, node_id=None, rel_type=None, direction="both", limit=100, **options):
- return [
- {
- "id": "r1",
- "source": "n1",
- "target": "n1",
- "type": "KNOWS",
- "properties": {},
- }
- ]
-
- monkeypatch.setattr(
- "semantica.graph_store.methods._get_store",
- lambda: FakeGraphStore(),
- )
- monkeypatch.setattr(
- "semantica.graph_store.get_nodes",
- lambda **kwargs: [
- {
- "id": "n1",
- "type": "Person",
- "name": "Alice",
- "properties": {"name": "Alice"},
- }
- ],
- )
- monkeypatch.setattr(
- "semantica.graph_store.methods.get_nodes",
- lambda **kwargs: [
- {
- "id": "n1",
- "type": "Person",
- "name": "Alice",
- "properties": {"name": "Alice"},
- }
- ],
- )
- monkeypatch.setattr(
- "semantica.graph_store.get_relationships",
- lambda **kwargs: [
- {
- "id": "r1",
- "source": "n1",
- "target": "n1",
- "type": "KNOWS",
- "properties": {},
- }
- ],
- )
- monkeypatch.setattr(
- "semantica.graph_store.methods.get_relationships",
- lambda **kwargs: [
- {
- "id": "r1",
- "source": "n1",
- "target": "n1",
- "type": "KNOWS",
- "properties": {},
- }
- ],
- )
+ _patch_graph_store(monkeypatch, [STORE_RELATIONSHIP])
output_path = tmp_path / "export.json"
result = runner.invoke(cli_module.main, ["export", "--format", "json", "--output", str(output_path)])
@@ -2003,6 +2000,82 @@ class TestExport:
assert "Traceback" not in result.output
+class TestExportMultiFileFormats:
+ """Formats that write one file per collection, driven through the command.
+
+ ``arrow``, ``csv`` and ``parquet`` take ``--output`` as a base name and
+ write ``graph_entities.*``/``graph_relationships.*`` next to it, so the
+ path the command was given is never created. It used to report that path as
+ written anyway, hand stdout an empty file, and compress a base file that
+ did not exist.
+ """
+
+ @pytest.mark.parametrize("fmt,ext", sorted(MULTI_FILE_EXTENSIONS.items()))
+ def test_reports_the_files_it_wrote(self, runner, tmp_path, monkeypatch, fmt, ext):
+ _patch_graph_store(monkeypatch, [STORE_RELATIONSHIP])
+ target = tmp_path / f"graph{ext}"
+
+ result = runner.invoke(
+ cli_module.main, ["export", "--format", fmt, "--output", str(target)]
+ )
+
+ _ok(result)
+ names = sorted(p.name for p in tmp_path.iterdir())
+ assert names == [f"graph_entities{ext}", f"graph_relationships{ext}"], names
+ for name in names:
+ assert name in result.output, f"{name} not reported: {result.output!r}"
+ assert not target.exists(), (
+ "the command was asked for a base name; it must not be reported as "
+ "the artifact"
+ )
+
+ @pytest.mark.parametrize("fmt,ext", sorted(MULTI_FILE_EXTENSIONS.items()))
+ @pytest.mark.parametrize(
+ "flags,mention",
+ [([], "--output"), (["--compress"], "--compress")],
+ )
+ def test_a_single_destination_is_refused(
+ self, runner, tmp_path, monkeypatch, fmt, ext, flags, mention
+ ):
+ reads = _patch_graph_store(monkeypatch, [STORE_RELATIONSHIP])
+
+ result = runner.invoke(cli_module.main, ["export", "--format", fmt] + flags)
+
+ assert result.exit_code != 0, f"{flags} exited 0: {result.output!r}"
+ assert mention in result.output, f"{result.output!r} does not mention {mention}"
+ assert reads["nodes"] == 0, "the store was read before the format was refused"
+
+ def test_arrow_without_relationships_still_exports(
+ self, runner, tmp_path, monkeypatch
+ ):
+ """A graph whose nodes have no edges between them is not an error."""
+ _patch_graph_store(monkeypatch, [])
+ target = tmp_path / "graph.arrow"
+
+ result = runner.invoke(
+ cli_module.main, ["export", "--format", "arrow", "--output", str(target)]
+ )
+
+ _ok(result)
+ assert (tmp_path / "graph_entities.arrow").exists()
+ assert not (tmp_path / "graph_relationships.arrow").exists()
+
+ @pytest.mark.parametrize("fmt,ext", sorted(MULTI_FILE_EXTENSIONS.items()))
+ def test_an_empty_store_writes_nothing_and_says_so(
+ self, runner, tmp_path, monkeypatch, fmt, ext
+ ):
+ _patch_graph_store(monkeypatch, [], nodes=[])
+ target = tmp_path / f"graph{ext}"
+
+ result = runner.invoke(
+ cli_module.main, ["export", "--format", fmt, "--output", str(target)]
+ )
+
+ _ok(result)
+ assert "Nothing written" in result.output, result.output
+ assert not list(tmp_path.iterdir())
+
+
# ─── visualize ────────────────────────────────────────────────────────────────