Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

Merge branch 'main' into feat/add-apache-parquet-support

Mohd Kaif930be04fed48
diff --cc semantica/export/__init__.py
index 51e304ac,9b21b92b..4366544f
--- a/semantica/export/__init__.py
+++ b/semantica/export/__init__.py
@@@ -8,8 -9,8 +9,10 @@@ ArangoDB AQL formats
Algorithms Used:
RDF Export:
+ - RDF Serialization: Multiple format serialization (Turtle,
+ RDF/XML, JSON-LD, N-Triples, N3)
+ - RDF Serialization: Multiple format serialization (Turtle, RDF/XML,
+ JSON-LD, N-Triples, N3)
- Namespace Management: Namespace registration, conflict resolution,
declaration generation
- RDF Validation: RDF syntax validation, triplet validation,
@@@ -19,8 -20,8 +22,10 @@@
- Format Conversion: Cross-format RDF conversion algorithms
LPG (Labeled Property Graph) Export:
+ - LPG Serialization: Labeled Property Graph format for Neo4j,
+ Memgraph, and similar databases
+ - LPG Serialization: Labeled Property Graph format for Neo4j, Memgraph,
+ and similar databases
- Node Label Assignment: Entity type to node label mapping
- Relationship Type Mapping: Relationship type to edge label conversion
- Property Serialization: Entity and relationship properties to LPG property format
@@@ -40,7 -51,8 +55,9 @@@ CSV Export
- Field Extraction: Dynamic field name extraction from data structures
- Delimiter Handling: Configurable delimiter support (comma, tab, semicolon)
- Header Generation: Automatic CSV header row generation
+ - Metadata Serialization: JSON string serialization for complex metadata fields
+ - Metadata Serialization: JSON string serialization for complex
+ metadata fields
- Multi-file Export: Knowledge graph split into multiple CSV files
(entities, relationships)
@@@ -62,20 -74,10 +79,21 @@@ OWL Export
- Turtle Serialization: OWL in Turtle format
- Class Hierarchy Export: Class definition and hierarchy serialization
- Property Export: Object and data property definition export
- - OWL 2.0 Feature Support: Advanced OWL features (cardinality, restrictions, etc.)
+ - OWL 2.0 Feature Support: Advanced OWL features (cardinality,
+ restrictions, etc.)
- Ontology Validation: OWL syntax and semantic validation
+Parquet Export:
+ - Parquet Serialization: Columnar storage format for analytics
+ - Schema Definition: Explicit Arrow schema for entities and
+ relationships
+ - Compression: Configurable compression (snappy, gzip, brotli,
+ zstd, lz4)
+ - Metadata Handling: Structured metadata as Parquet structs
+ - Analytics Integration: Compatible with pandas, Spark, Snowflake,
+ BigQuery, Databricks
+ - Batch Export: Efficient batch processing for large graphs
+
Vector Export:
- Vector Serialization: Multiple format support (JSON, NumPy, Binary, FAISS)
- Vector Store Integration: Format conversion for Weaviate, Qdrant, FAISS
@@@ -91,7 -93,8 +109,9 @@@ Report Generation
- Section Organization: Hierarchical report section organization
Key Features:
+ - Multiple export formats (RDF, JSON, CSV, Graph, YAML, OWL, Vector, LPG, Parquet)
+ - Multiple export formats (RDF, JSON, CSV, Graph, YAML, OWL, Vector,
+ LPG, ArangoDB AQL)
- Knowledge graph export with format auto-detection
- Report generation (HTML, Markdown, JSON, Text)
- Vector store integration
diff --cc semantica/export/arrow_exporter.py
index 85e4b1cf,d043a442..d2e3a4d1
--- a/semantica/export/arrow_exporter.py
+++ b/semantica/export/arrow_exporter.py
@@@ -28,60 -30,17 +30,65 @@@ import jso
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+if TYPE_CHECKING:
+ import pyarrow as pa # noqa: F401
+
++
+ if TYPE_CHECKING:
+ import pyarrow as pa
+
try:
import pyarrow as pa # noqa: F811
import pyarrow.ipc as ipc
ARROW_AVAILABLE = True
+
+ # Explicit Arrow Schemas (no inference)
+ ENTITY_SCHEMA = pa.schema(
+ [
+ pa.field("id", pa.string(), nullable=False),
+ pa.field("text", pa.string(), nullable=True),
+ pa.field("type", pa.string(), nullable=True),
+ pa.field("confidence", pa.float64(), nullable=True),
+ pa.field("start", pa.int64(), nullable=True),
+ pa.field("end", pa.int64(), nullable=True),
+ pa.field(
+ "metadata",
+ pa.struct(
+ [
+ pa.field("keys", pa.list_(pa.string())),
+ pa.field("values", pa.list_(pa.string())),
+ ]
+ ),
+ nullable=True,
+ ),
+ ]
+ )
+
+ RELATIONSHIP_SCHEMA = pa.schema(
+ [
+ pa.field("id", pa.string(), nullable=False),
+ pa.field("source_id", pa.string(), nullable=False),
+ pa.field("target_id", pa.string(), nullable=False),
+ pa.field("type", pa.string(), nullable=True),
+ pa.field("confidence", pa.float64(), nullable=True),
+ pa.field(
+ "metadata",
+ pa.struct(
+ [
+ pa.field("keys", pa.list_(pa.string())),
+ pa.field("values", pa.list_(pa.string())),
+ ]
+ ),
+ nullable=True,
+ ),
+ ]
+ )
except ImportError:
ARROW_AVAILABLE = False
+ ENTITY_SCHEMA = None
+ RELATIONSHIP_SCHEMA = None
+ pa = None # Set pa to None when not available
from ..utils.exceptions import ProcessingError, ValidationError
from ..utils.helpers import ensure_directory
@@@ -321,7 -328,7 +376,8 @@@ class ArrowExporter
try:
confidence = float(confidence)
except (TypeError, ValueError):
+ msg = (
+ self.logger.warning(
f"Invalid confidence value for entity {i}: "
f"{confidence}. Setting to None."
)
@@@ -429,7 -435,7 +485,8 @@@
try:
confidence = float(confidence)
except (TypeError, ValueError):
+ msg = (
+ self.logger.warning(
f"Invalid confidence value for relationship {i}: "
f"{confidence}. Setting to None."
)
diff --cc semantica/export/methods.py
index 72ba7470,0138199c..8fafc9af
--- a/semantica/export/methods.py
+++ b/semantica/export/methods.py
@@@ -159,8 -153,9 +159,9 @@@ Example Usage
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional, Union
-from ..utils.exceptions import ConfigurationError, ProcessingError
+from ..utils.exceptions import ProcessingError
from ..utils.logging import get_logger
+ from .arango_aql_exporter import ArangoAQLExporter
from .arrow_exporter import ArrowExporter
from .config import export_config
from .csv_exporter import CSVExporter