Graph-Native Infrastructure for Context and Accountable AI Systems
Python agent-memoryaiai-governanceai-infrastructureartificial-intelligencecontext-engineeringcontext-graphsdata-engineeringdecision-intelligencedeveloper-toolsexplainable-aigenerative-aigraph-ragknowledge-graphllmontologyprovenancepythonreasoningsemantic-search
fix: rename contributing/license pages to avoid Mintlify reserved slug conflict
KaifAhmad1453eeb7ca9a5
diff --git a/.gitattributes b/.gitattributesindex 5595f17b..aad0da1d 100644--- a/.gitattributes+++ b/.gitattributes@@ -1,3 +1,11 @@+# Line endings — force LF so Mintlify/Linux CI parses frontmatter correctly+* text=auto eol=lf+*.md text eol=lf+*.json text eol=lf+*.yml text eol=lf+*.yaml text eol=lf+*.py text eol=lf+# Linguist documentation and generated files# This ensures GitHub language statistics reflect the core Python codediff --git a/docs/citation.md b/docs/citation.mdindex a937117e..fd4a6728 100644--- a/docs/citation.md+++ b/docs/citation.md@@ -62,7 +62,7 @@ Published research using Semantica? [Let us know](https://github.com/semantica-a## See Also<CardGroup cols={2}>- <Card title="License" icon="file-contract" href="license">+ <Card title="License" icon="file-contract" href="project-license">MIT License details.</Card><Card title="Community" icon="users" href="community">diff --git a/docs/community-projects.md b/docs/community-projects.mdindex 9ef10b7f..c09e46f6 100644--- a/docs/community-projects.md+++ b/docs/community-projects.md@@ -55,7 +55,7 @@ Have a project using Semantica? [Submit it on GitHub](https://github.com/semantiSee the [Contributing Guide](contributing) for full details.<CardGroup cols={2}>- <Card title="Contributing" icon="code-pull-request" href="contributing">+ <Card title="Contributing" icon="code-pull-request" href="contributing-guide">Submit code, docs, or tests.</Card><Card title="GitHub Issues" icon="circle-dot" href="https://github.com/semantica-agi/semantica/issues">diff --git a/docs/community.md b/docs/community.mdindex ab624904..17614a34 100644--- a/docs/community.md+++ b/docs/community.md@@ -61,7 +61,7 @@ See the [Contributing Guide](contributing) for full details.## See Also<CardGroup cols={2}>- <Card title="Contributing" icon="code-pull-request" href="contributing">+ <Card title="Contributing" icon="code-pull-request" href="contributing-guide">Step-by-step guide for submitting PRs.</Card><Card title="FAQ" icon="circle-question" href="faq">diff --git a/docs/contributing.md b/docs/contributing-guide.mdsimilarity index 100%rename from docs/contributing.mdrename to docs/contributing-guide.mddiff --git a/docs/docs.json b/docs/docs.jsonindex e9324bf4..2be3d248 100644--- a/docs/docs.json+++ b/docs/docs.json@@ -194,10 +194,10 @@"pages": ["community","community-projects",- "contributing",+ "contributing-guide","governance","citation",- "license"+ "project-license"]}]diff --git a/docs/faq.md b/docs/faq.mdindex 43572513..88af3cac 100644--- a/docs/faq.md+++ b/docs/faq.md@@ -235,7 +235,7 @@ pip install --upgrade semantica<Card title="GitHub Issues" icon="github" href="https://github.com/semantica-agi/semantica/issues">Bug reports and feature requests.</Card>- <Card title="Contributing" icon="code-pull-request" href="contributing">+ <Card title="Contributing" icon="code-pull-request" href="contributing-guide">Help improve Semantica.</Card></CardGroup>diff --git a/docs/governance.md b/docs/governance.mdindex f769d67d..791279d8 100644--- a/docs/governance.md+++ b/docs/governance.md@@ -98,7 +98,7 @@ MIT License — see [LICENSE](https://github.com/semantica-agi/semantica/blob/ma## See Also<CardGroup cols={2}>- <Card title="Contributing" icon="code-pull-request" href="contributing">+ <Card title="Contributing" icon="code-pull-request" href="contributing-guide">How to submit changes.</Card><Card title="Community" icon="users" href="community">diff --git a/docs/license.md b/docs/project-license.mdsimilarity index 99%rename from docs/license.mdrename to docs/project-license.mdindex 6e3871f0..bad87bef 100644--- a/docs/license.md+++ b/docs/project-license.md@@ -83,7 +83,7 @@ By contributing to Semantica, you agree that your contributions will be licensed## See Also<CardGroup cols={2}>- <Card title="Contributing" icon="code-pull-request" href="contributing">+ <Card title="Contributing" icon="code-pull-request" href="contributing-guide">How to contribute to the project.</Card><Card title="Citation" icon="quote-left" href="citation">diff --git a/examples/parquet_export_example.py b/examples/parquet_export_example.pyindex 5dcb0db0..0f47e82c 100644--- a/examples/parquet_export_example.py+++ b/examples/parquet_export_example.py@@ -1,213 +1,213 @@-"""-Apache Parquet Exporter - Example Usage--This script demonstrates how to use the ParquetExporter to export-knowledge graphs, entities, and relationships to Apache Parquet format.-"""--import tempfile-from pathlib import Path--from semantica.export import ParquetExporter, export_parquet---def main():- print("=" * 70)- print("Apache Parquet Exporter - Example Usage")- print("=" * 70)-- # Create a temporary directory for outputs- temp_dir = Path(tempfile.mkdtemp())- print(f"\n📁 Output directory: {temp_dir}\n")-- # Sample data- entities = [- {- "id": "e1",- "text": "Alice",- "type": "Person",- "confidence": 0.95,- "start": 0,- "end": 5,- "metadata": {"age": 30, "city": "New York"},- },- {- "id": "e2",- "text": "Acme Corp",- "type": "Organization",- "confidence": 0.88,- "start": 10,- "end": 19,- "metadata": {"location": "NY", "employees": 100},- },- {- "id": "e3",- "text": "Bob",- "type": "Person",- "confidence": 0.92,- "metadata": {"age": 35, "department": "Engineering"},- },- ]-- relationships = [- {- "id": "r1",- "source_id": "e1",- "target_id": "e2",- "type": "WORKS_FOR",- "confidence": 0.90,- "metadata": {"role": "Engineer", "since": 2020},- },- {- "id": "r2",- "source_id": "e3",- "target_id": "e2",- "type": "WORKS_FOR",- "confidence": 0.85,- "metadata": {"role": "Manager", "since": 2018},- },- ]-- knowledge_graph = {- "entities": entities,- "relationships": relationships,- "metadata": {"version": "1.0", "created": "2024-01-01"},- }-- # Example 1: Export entities using ParquetExporter class- print("Example 1: Export entities to Parquet")- print("-" * 70)- exporter = ParquetExporter(compression="snappy")- entities_path = temp_dir / "entities.parquet"- exporter.export_entities(entities, entities_path)- print(f"✓ Entities exported to: {entities_path}")- print(f" File size: {entities_path.stat().st_size} bytes\n")-- # Example 2: Export relationships- print("Example 2: Export relationships to Parquet")- print("-" * 70)- rels_path = temp_dir / "relationships.parquet"- exporter.export_relationships(relationships, rels_path)- print(f"✓ Relationships exported to: {rels_path}")- print(f" File size: {rels_path.stat().st_size} bytes\n")-- # Example 3: Export complete knowledge graph- print("Example 3: Export knowledge graph to multiple Parquet files")- print("-" * 70)- kg_base_path = temp_dir / "knowledge_graph"- exporter.export_knowledge_graph(knowledge_graph, kg_base_path)- kg_entities = temp_dir / "knowledge_graph_entities.parquet"- kg_rels = temp_dir / "knowledge_graph_relationships.parquet"- print("✓ Knowledge graph exported to:")- print(f" - {kg_entities} ({kg_entities.stat().st_size} bytes)")- print(f" - {kg_rels} ({kg_rels.stat().st_size} bytes)\n")-- # Example 4: Using convenience function- print("Example 4: Using export_parquet convenience function")- print("-" * 70)- conv_path = temp_dir / "convenience_export.parquet"- export_parquet(entities, conv_path, compression="gzip")- print(f"✓ Exported using convenience function: {conv_path}")- print(f" File size: {conv_path.stat().st_size} bytes\n")-- # Example 5: Different compression codecs- print("Example 5: Compare compression codecs")- print("-" * 70)-- # Create larger dataset for meaningful comparison- large_entities = entities * 50-- compression_codecs = ["snappy", "gzip", "brotli", "zstd", "lz4", "none"]- sizes = {}-- for codec in compression_codecs:- codec_exporter = ParquetExporter(compression=codec)- codec_path = temp_dir / f"entities_{codec}.parquet"- codec_exporter.export_entities(large_entities, codec_path)- sizes[codec] = codec_path.stat().st_size- print(f" {codec:8} - {sizes[codec]:,} bytes")-- print()-- # Example 6: Load Parquet with pandas (if available)- print("Example 6: Loading Parquet files with pandas")- print("-" * 70)- try:- import pandas as pd-- df = pd.read_parquet(entities_path)- print("✓ Loaded entities as pandas DataFrame")- print(f" Shape: {df.shape}")- print(f" Columns: {list(df.columns)}")- print("\nFirst few rows:")- print(df.head())- print()-- except ImportError:- print("⚠ pandas not installed - skipping pandas example\n")-- # Example 7: Load Parquet with pyarrow- print("Example 7: Loading Parquet files with pyarrow")- print("-" * 70)- try:- import pyarrow.parquet as pq-- table = pq.read_table(entities_path)- print("✓ Loaded entities as Arrow Table")- print(f" Rows: {table.num_rows}")- print(f" Columns: {table.num_columns}")- print(" Schema:")- for i, field in enumerate(table.schema):- print(f" - {field.name}: {field.type}")- print()-- except ImportError:- print("⚠ pyarrow not installed - skipping pyarrow example\n")-- # Example 8: Schema validation- print("Example 8: Explicit schema validation")- print("-" * 70)- try:- import pyarrow.parquet as pq-- # Read parquet file and verify schema- table = pq.read_table(entities_path)-- print("✓ Schema validation:")- print(f" - ID column type: {table.schema.field('id').type}")- print(f" - Text column type: {table.schema.field('text').type}")- print(f" - Confidence column type: {table.schema.field('confidence').type}")- print(f" - Metadata column type: {table.schema.field('metadata').type}")- print()-- # Verify metadata structure- metadata_field = table.schema.field("metadata")- print(" Metadata structure:")- if hasattr(metadata_field.type, "num_fields"):- for i in range(metadata_field.type.num_fields):- subfield = metadata_field.type.field(i)- print(f" - {subfield.name}: {subfield.type}")- print()-- except Exception as e:- print(f"⚠ Schema validation error: {e}\n")-- # Summary- print("=" * 70)- print("Summary")- print("=" * 70)- print("✓ All examples completed successfully")- print(f"✓ Output directory: {temp_dir}")- print(f"✓ Files created: {len(list(temp_dir.glob('*.parquet')))}")- print("\nKey Features:")- print(" - Columnar storage optimized for analytics")- print(" - Multiple compression options (snappy, gzip, brotli, zstd, lz4)")- print(" - Compatible with pandas, Spark, Snowflake, BigQuery, Databricks")- print(" - Explicit schemas for type safety")- print(" - Structured metadata handling")- print("\nFor more information, see the Semantica documentation.")- print("=" * 70)---if __name__ == "__main__":- main()+"""+Apache Parquet Exporter - Example Usage++This script demonstrates how to use the ParquetExporter to export+knowledge graphs, entities, and relationships to Apache Parquet format.+"""++import tempfile+from pathlib import Path++from semantica.export import ParquetExporter, export_parquet+++def main():+ print("=" * 70)+ print("Apache Parquet Exporter - Example Usage")+ print("=" * 70)++ # Create a temporary directory for outputs+ temp_dir = Path(tempfile.mkdtemp())+ print(f"\n📁 Output directory: {temp_dir}\n")++ # Sample data+ entities = [+ {+ "id": "e1",+ "text": "Alice",+ "type": "Person",+ "confidence": 0.95,+ "start": 0,+ "end": 5,+ "metadata": {"age": 30, "city": "New York"},+ },+ {+ "id": "e2",+ "text": "Acme Corp",+ "type": "Organization",+ "confidence": 0.88,+ "start": 10,+ "end": 19,+ "metadata": {"location": "NY", "employees": 100},+ },+ {+ "id": "e3",+ "text": "Bob",+ "type": "Person",+ "confidence": 0.92,+ "metadata": {"age": 35, "department": "Engineering"},+ },+ ]++ relationships = [+ {+ "id": "r1",+ "source_id": "e1",+ "target_id": "e2",+ "type": "WORKS_FOR",+ "confidence": 0.90,+ "metadata": {"role": "Engineer", "since": 2020},+ },+ {+ "id": "r2",+ "source_id": "e3",+ "target_id": "e2",+ "type": "WORKS_FOR",+ "confidence": 0.85,+ "metadata": {"role": "Manager", "since": 2018},+ },+ ]++ knowledge_graph = {+ "entities": entities,+ "relationships": relationships,+ "metadata": {"version": "1.0", "created": "2024-01-01"},+ }++ # Example 1: Export entities using ParquetExporter class+ print("Example 1: Export entities to Parquet")+ print("-" * 70)+ exporter = ParquetExporter(compression="snappy")+ entities_path = temp_dir / "entities.parquet"+ exporter.export_entities(entities, entities_path)+ print(f"✓ Entities exported to: {entities_path}")+ print(f" File size: {entities_path.stat().st_size} bytes\n")++ # Example 2: Export relationships+ print("Example 2: Export relationships to Parquet")+ print("-" * 70)+ rels_path = temp_dir / "relationships.parquet"+ exporter.export_relationships(relationships, rels_path)+ print(f"✓ Relationships exported to: {rels_path}")+ print(f" File size: {rels_path.stat().st_size} bytes\n")++ # Example 3: Export complete knowledge graph+ print("Example 3: Export knowledge graph to multiple Parquet files")+ print("-" * 70)+ kg_base_path = temp_dir / "knowledge_graph"+ exporter.export_knowledge_graph(knowledge_graph, kg_base_path)+ kg_entities = temp_dir / "knowledge_graph_entities.parquet"+ kg_rels = temp_dir / "knowledge_graph_relationships.parquet"+ print("✓ Knowledge graph exported to:")+ print(f" - {kg_entities} ({kg_entities.stat().st_size} bytes)")+ print(f" - {kg_rels} ({kg_rels.stat().st_size} bytes)\n")++ # Example 4: Using convenience function+ print("Example 4: Using export_parquet convenience function")+ print("-" * 70)+ conv_path = temp_dir / "convenience_export.parquet"+ export_parquet(entities, conv_path, compression="gzip")+ print(f"✓ Exported using convenience function: {conv_path}")+ print(f" File size: {conv_path.stat().st_size} bytes\n")++ # Example 5: Different compression codecs+ print("Example 5: Compare compression codecs")+ print("-" * 70)++ # Create larger dataset for meaningful comparison+ large_entities = entities * 50++ compression_codecs = ["snappy", "gzip", "brotli", "zstd", "lz4", "none"]+ sizes = {}++ for codec in compression_codecs:+ codec_exporter = ParquetExporter(compression=codec)+ codec_path = temp_dir / f"entities_{codec}.parquet"+ codec_exporter.export_entities(large_entities, codec_path)+ sizes[codec] = codec_path.stat().st_size+ print(f" {codec:8} - {sizes[codec]:,} bytes")++ print()++ # Example 6: Load Parquet with pandas (if available)+ print("Example 6: Loading Parquet files with pandas")+ print("-" * 70)+ try:+ import pandas as pd++ df = pd.read_parquet(entities_path)+ print("✓ Loaded entities as pandas DataFrame")+ print(f" Shape: {df.shape}")+ print(f" Columns: {list(df.columns)}")+ print("\nFirst few rows:")+ print(df.head())+ print()++ except ImportError:+ print("⚠ pandas not installed - skipping pandas example\n")++ # Example 7: Load Parquet with pyarrow+ print("Example 7: Loading Parquet files with pyarrow")+ print("-" * 70)+ try:+ import pyarrow.parquet as pq++ table = pq.read_table(entities_path)+ print("✓ Loaded entities as Arrow Table")+ print(f" Rows: {table.num_rows}")+ print(f" Columns: {table.num_columns}")+ print(" Schema:")+ for i, field in enumerate(table.schema):+ print(f" - {field.name}: {field.type}")+ print()++ except ImportError:+ print("⚠ pyarrow not installed - skipping pyarrow example\n")++ # Example 8: Schema validation+ print("Example 8: Explicit schema validation")+ print("-" * 70)+ try:+ import pyarrow.parquet as pq++ # Read parquet file and verify schema+ table = pq.read_table(entities_path)++ print("✓ Schema validation:")+ print(f" - ID column type: {table.schema.field('id').type}")+ print(f" - Text column type: {table.schema.field('text').type}")+ print(f" - Confidence column type: {table.schema.field('confidence').type}")+ print(f" - Metadata column type: {table.schema.field('metadata').type}")+ print()++ # Verify metadata structure+ metadata_field = table.schema.field("metadata")+ print(" Metadata structure:")+ if hasattr(metadata_field.type, "num_fields"):+ for i in range(metadata_field.type.num_fields):+ subfield = metadata_field.type.field(i)+ print(f" - {subfield.name}: {subfield.type}")+ print()++ except Exception as e:+ print(f"⚠ Schema validation error: {e}\n")++ # Summary+ print("=" * 70)+ print("Summary")+ print("=" * 70)+ print("✓ All examples completed successfully")+ print(f"✓ Output directory: {temp_dir}")+ print(f"✓ Files created: {len(list(temp_dir.glob('*.parquet')))}")+ print("\nKey Features:")+ print(" - Columnar storage optimized for analytics")+ print(" - Multiple compression options (snappy, gzip, brotli, zstd, lz4)")+ print(" - Compatible with pandas, Spark, Snowflake, BigQuery, Databricks")+ print(" - Explicit schemas for type safety")+ print(" - Structured metadata handling")+ print("\nFor more information, see the Semantica documentation.")+ print("=" * 70)+++if __name__ == "__main__":+ main()diff --git a/semantica/export/arango_aql_exporter.py b/semantica/export/arango_aql_exporter.pyindex 712cb1d3..69140481 100644--- a/semantica/export/arango_aql_exporter.py+++ b/semantica/export/arango_aql_exporter.py@@ -1,641 +1,641 @@-"""-ArangoDB AQL Export Module--This module provides comprehensive AQL export capabilities for the Semantica framework,-enabling export to ArangoDB multi-model graph databases.--Key Features:- - AQL format export for ArangoDB- - INSERT statement generation for vertex and edge collections- - Configurable collection names- - Entity and relationship identifier preservation- - Batch insert support for performance- - Proper string escaping and special character handling--Example Usage:- >>> from semantica.export import ArangoAQLExporter- >>> exporter = ArangoAQLExporter()- >>> exporter.export_knowledge_graph(kg, "output.aql")- >>> # With custom collection names- >>> exporter = ArangoAQLExporter(- ... vertex_collection="nodes",- ... edge_collection="links"- ... )- >>> exporter.export(kg, "graph.aql")-"""--import json-from pathlib import Path-from typing import Any, Dict, List, Optional, Union--from ..utils.helpers import ensure_directory-from ..utils.logging import get_logger-from ..utils.progress_tracker import get_progress_tracker---class ArangoAQLExporter:- """- ArangoDB AQL exporter for knowledge graphs.-- This class provides comprehensive AQL export functionality for knowledge graphs,- supporting export to ArangoDB multi-model databases via AQL INSERT statements.-- Features:- - AQL INSERT statement generation for vertices and edges- - Configurable collection names- - Entity and relationship identifier preservation- - Batch insert support for performance- - Proper string escaping and special character handling- - Support for nested properties via JSON serialization-- Example Usage:- >>> exporter = ArangoAQLExporter(- ... vertex_collection="entities",- ... edge_collection="relationships"- ... )- >>> exporter.export_knowledge_graph(kg, "output.aql")- """-- def __init__(- self,- vertex_collection: str = "vertices",- edge_collection: str = "edges",- batch_size: int = 1000,- include_collection_creation: bool = True,- config: Optional[Dict[str, Any]] = None,- **kwargs,- ):- """- Initialize ArangoDB AQL exporter.-- Sets up the exporter with collection names and batch processing options.-- Args:- vertex_collection: Name of the vertex collection- (default: "vertices")- edge_collection: Name of the edge collection (default: "edges")- batch_size: Batch size for INSERT operations (default: 1000)- include_collection_creation: Whether to include collection- creation statements (default: True)- config: Optional configuration dictionary (merged with kwargs)- **kwargs: Additional configuration options- """- self.logger = get_logger("arango_aql_exporter")- self.config = config or {}- self.config.update(kwargs)-- # Validate collection names- self._validate_collection_name(vertex_collection, "vertex_collection")- self._validate_collection_name(edge_collection, "edge_collection")-- # AQL export configuration- self.vertex_collection = vertex_collection- self.edge_collection = edge_collection- self.batch_size = batch_size- self.include_collection_creation = include_collection_creation-- # Initialize progress tracker- self.progress_tracker = get_progress_tracker()-- self.logger.debug(- f"ArangoDB AQL exporter initialized: "- f"vertex_collection={vertex_collection}, "- f"edge_collection={edge_collection}, batch_size={batch_size}"- )-- def export(- self, knowledge_graph: Dict[str, Any], file_path: Union[str, Path], **options- ) -> None:- """- Export knowledge graph to ArangoDB AQL format.-- This method exports a knowledge graph to AQL INSERT statements that can- be imported into ArangoDB multi-model databases.-- Args:- knowledge_graph: Knowledge graph dictionary containing:- - entities: List of entity dictionaries- - relationships: List of relationship dictionaries- - nodes: List of node dictionaries (optional, alternative to- entities)- - edges: List of edge dictionaries (optional, alternative to- relationships)- file_path: Output AQL file path- **options: Additional export options:- - vertex_collection: Override default vertex collection name- - edge_collection: Override default edge collection name-- Example:- >>> kg = {- ... "entities": [...],- ... "relationships": [...]- ... }- >>> exporter.export(kg, "graph.aql")- """- file_path = Path(file_path)- ensure_directory(file_path.parent)-- tracking_id = self.progress_tracker.start_tracking(- file=str(file_path),- module="export",- submodule="ArangoAQLExporter",- message=f"Exporting knowledge graph to ArangoDB AQL format: {file_path}",- )-- try:- # Override collection names if provided in options- vertex_collection = options.pop("vertex_collection", self.vertex_collection)- edge_collection = options.pop("edge_collection", self.edge_collection)-- # Validate overridden collection names- if vertex_collection != self.vertex_collection:- self._validate_collection_name(vertex_collection, "vertex_collection")- if edge_collection != self.edge_collection:- self._validate_collection_name(edge_collection, "edge_collection")-- # Generate AQL statements- aql_statements = self._generate_aql_statements(- knowledge_graph, vertex_collection, edge_collection, **options- )-- # Write to file- with open(file_path, "w", encoding="utf-8") as f:- f.write("\n".join(aql_statements))-- self.progress_tracker.stop_tracking(- tracking_id,- status="completed",- message=f"Exported {len(aql_statements)} AQL statements",- )- self.logger.info(- f"Exported knowledge graph to ArangoDB AQL format: " f"{file_path}"- )-- except Exception as e:- self.progress_tracker.stop_tracking(- tracking_id, status="failed", message=str(e)- )- raise-- def _generate_aql_statements(- self,- knowledge_graph: Dict[str, Any],- vertex_collection: str,- edge_collection: str,- **options,- ) -> List[str]:- """- Generate AQL INSERT statements from knowledge graph.-- Args:- knowledge_graph: Knowledge graph dictionary- vertex_collection: Vertex collection name- edge_collection: Edge collection name- **options: Additional options-- Returns:- List of AQL statement strings- """- statements = []-- # Add collection creation statements if requested- if self.include_collection_creation:- statements.extend(- self._generate_collection_creation(vertex_collection, edge_collection)- )-- # Extract entities and relationships- entities = knowledge_graph.get("entities", [])- relationships = knowledge_graph.get("relationships", [])- nodes = knowledge_graph.get("nodes", entities)- edges = knowledge_graph.get("edges", relationships)-- # Use nodes/edges if entities/relationships are empty- if not entities and nodes:- entities = nodes- if not relationships and edges:- relationships = edges-- # Generate vertex INSERT statements- vertex_statements = self._generate_vertex_inserts(entities, vertex_collection)- statements.extend(vertex_statements)-- # Generate edge INSERT statements- edge_statements = self._generate_edge_inserts(- relationships, edge_collection, vertex_collection- )- statements.extend(edge_statements)-- return statements-- def _generate_collection_creation(- self, vertex_collection: str, edge_collection: str- ) -> List[str]:- """- Generate AQL collection creation statements.-- Args:- vertex_collection: Vertex collection name- edge_collection: Edge collection name-- Returns:- List of collection creation statements- """- statements = [- "// Create vertex collection if it doesn't exist",- f"// db._createDocumentCollection('{vertex_collection}');",- "",- "// Create edge collection if it doesn't exist",- f"// db._createEdgeCollection('{edge_collection}');",- "",- ]- return statements-- def _generate_vertex_inserts(- self, vertices: List[Dict[str, Any]], collection: str- ) -> List[str]:- """- Generate AQL INSERT statements for vertices.-- Args:- vertices: List of vertex/entity dictionaries- collection: Vertex collection name-- Returns:- List of AQL INSERT statements- """- statements = []-- # Add header comment- statements.append(f"// Inserting {len(vertices)} vertices into {collection}")- statements.append("")-- # Process vertices in batches- for i in range(0, len(vertices), self.batch_size):- batch = vertices[i : i + self.batch_size]- batch_statement = self._create_vertex_batch_insert(batch, collection)- statements.append(batch_statement)- statements.append("")-- return statements-- def _create_vertex_batch_insert(- self, vertices: List[Dict[str, Any]], collection: str- ) -> str:- """- Create a batch INSERT statement for vertices.-- Args:- vertices: Batch of vertex dictionaries- collection: Vertex collection name-- Returns:- AQL INSERT statement- """- if not vertices:- return ""-- # Build document list- documents = []- for idx, vertex in enumerate(vertices):- doc = self._convert_vertex_to_document(vertex, idx)- documents.append(doc)-- # Format as AQL- docs_json = json.dumps(documents, indent=2, ensure_ascii=False)- statement = f"FOR doc IN {docs_json}\n INSERT doc INTO {collection}"-- return statement-- def _convert_vertex_to_document(- self, vertex: Dict[str, Any], idx: int- ) -> Dict[str, Any]:- """- Convert a vertex/entity to an ArangoDB document.-- Additional properties from the input entity are preserved as-is in the- document root (not flattened or merged). Nested dictionaries and lists- are preserved as JSON-serializable structures. The 'properties' field,- if present, is also preserved as-is rather than being flattened into- the document root.-- Args:- vertex: Vertex/entity dictionary- idx: Index for generating fallback IDs-- Returns:- ArangoDB document dictionary- """- document = {}-- # Set _key from id or generate one- vertex_id = vertex.get("id") or vertex.get("entity_id") or f"vertex_{idx}"- document["_key"] = self._sanitize_key(str(vertex_id))-- # Add original ID if different from _key- if str(vertex_id) != document["_key"]:- document["original_id"] = str(vertex_id)-- # Add type/label information- vertex_type = vertex.get("type") or vertex.get("entity_type", "Entity")- document["type"] = vertex_type-- # Add name/label- label = (- vertex.get("label")- or vertex.get("name")- or vertex.get("text")- or document["_key"]- )- document["name"] = label-- # Add all other properties- for key, value in vertex.items():- if key not in [- "_key",- "_id",- "_rev",- "id",- "entity_id",- "type",- "entity_type",- "label",- "name",- "text",- ]:- # Handle nested dictionaries and lists- if isinstance(value, (dict, list)):- document[key] = value- elif value is not None:- document[key] = value-- return document-- def _generate_edge_inserts(- self, edges: List[Dict[str, Any]], collection: str, vertex_collection: str- ) -> List[str]:- """- Generate AQL INSERT statements for edges.-- Args:- edges: List of edge/relationship dictionaries- collection: Edge collection name- vertex_collection: Vertex collection name for _from/_to references-- Returns:- List of AQL INSERT statements- """- statements = []-- # Add header comment- statements.append(- f"// Attempting to insert {len(edges)} edges into {collection}"- )- statements.append("")-- # Process edges in batches- for i in range(0, len(edges), self.batch_size):- batch = edges[i : i + self.batch_size]- batch_statement = self._create_edge_batch_insert(- batch, collection, vertex_collection- )- if batch_statement: # Only add non-empty statements- statements.append(batch_statement)- statements.append("")-- return statements-- def _create_edge_batch_insert(- self, edges: List[Dict[str, Any]], collection: str, vertex_collection: str- ) -> str:- """- Create a batch INSERT statement for edges.-- Args:- edges: Batch of edge dictionaries- collection: Edge collection name- vertex_collection: Vertex collection name for _from/_to references-- Returns:- AQL INSERT statement- """- # Build document list- documents = []- for idx, edge in enumerate(edges):- doc = self._convert_edge_to_document(edge, idx, vertex_collection)- if doc: # Only add valid edges- documents.append(doc)-- if not documents:- return ""-- # Format as AQL- docs_json = json.dumps(documents, indent=2, ensure_ascii=False)- statement = f"FOR doc IN {docs_json}\n INSERT doc INTO {collection}"-- return statement-- def _convert_edge_to_document(- self, edge: Dict[str, Any], idx: int, vertex_collection: str- ) -> Optional[Dict[str, Any]]:- """- Convert an edge/relationship to an ArangoDB edge document.-- Args:- edge: Edge/relationship dictionary- idx: Index for generating fallback IDs- vertex_collection: Vertex collection name for _from/_to references-- Returns:- ArangoDB edge document dictionary, or None if source/target missing- """- # Extract source and target- source_id = edge.get("source") or edge.get("source_id")- target_id = edge.get("target") or edge.get("target_id")-- if not source_id or not target_id:- self.logger.warning(- f"Skipping edge {idx}: missing source or target "- f"(source={source_id}, target={target_id})"- )- return None-- document = {}-- # Set _key from id or generate one- edge_id = edge.get("id") or edge.get("relationship_id") or f"edge_{idx}"- document["_key"] = self._sanitize_key(str(edge_id))-- # Set _from and _to (required for edges in ArangoDB)- document["_from"] = f"{vertex_collection}/{self._sanitize_key(str(source_id))}"- document["_to"] = f"{vertex_collection}/{self._sanitize_key(str(target_id))}"-- # Add original ID if different from _key- if str(edge_id) != document["_key"]:- document["original_id"] = str(edge_id)-- # Add relationship type- rel_type = edge.get("type") or edge.get("relationship_type", "RELATED_TO")- document["type"] = rel_type-- # Add all other properties- for key, value in edge.items():- if key not in [- "_key",- "_id",- "_rev",- "_from",- "_to",- "id",- "relationship_id",- "source",- "source_id",- "target",- "target_id",- "type",- "relationship_type",- ]:- # Handle nested dictionaries and lists- if isinstance(value, (dict, list)):- document[key] = value- elif value is not None:- document[key] = value-- return document-- def _validate_collection_name(self, name: str, param_name: str) -> None:- """- Validate an ArangoDB collection name.-- ArangoDB collection names must:- - Start with a letter or underscore- - Contain only alphanumeric characters, hyphens, and underscores- - Not exceed 256 characters-- Args:- name: Collection name to validate- param_name: Parameter name for error messages-- Raises:- ValueError: If the collection name is invalid- """- if not name:- raise ValueError(f"{param_name} cannot be empty")-- if len(name) > 256:- raise ValueError(- f"{param_name} '{name}' exceeds maximum length of 256 characters"- )-- # Check first character- if not (name[0].isalpha() or name[0] == "_"):- raise ValueError(- f"{param_name} '{name}' must start with a letter or underscore"- )-- # Check remaining characters- for char in name:- if not (char.isalnum() or char in ("-", "_")):- raise ValueError(- f"{param_name} '{name}' contains invalid character "- f"'{char}'. Only alphanumeric characters, hyphens, and "- "underscores are allowed."- )-- def _sanitize_key(self, key: str) -> str:- """- Sanitize a key for use as ArangoDB _key.-- ArangoDB _key must contain only alphanumeric characters, hyphens,- and underscores. It cannot start with an underscore (unless it's- a system collection).-- Args:- key: Original key string-- Returns:- Sanitized key string- """- # Replace invalid characters with underscores- sanitized = ""- for char in key:- if char.isalnum() or char in ("-", "_"):- sanitized += char- else:- sanitized += "_"-- # Ensure key doesn't start with underscore- if sanitized.startswith("_"):- sanitized = "k" + sanitized-- # Ensure key is not empty- if not sanitized:- sanitized = "key"-- return sanitized-- def export_knowledge_graph(- self, knowledge_graph: Dict[str, Any], file_path: Union[str, Path], **options- ) -> None:- """- Export knowledge graph to ArangoDB AQL format.-- Convenience method that calls export().-- Args:- knowledge_graph: Knowledge graph dictionary- file_path: Output AQL file path- **options: Additional export options-- Example:- >>> kg = {- ... "entities": [...],- ... "relationships": [...]- ... }- >>> exporter.export_knowledge_graph(kg, "output.aql")- """- self.export(knowledge_graph, file_path, **options)-- def export_entities(- self, entities: List[Dict[str, Any]], file_path: Union[str, Path], **options- ) -> None:- """- Export entities to ArangoDB AQL format (vertices only).-- Args:- entities: List of entity dictionaries- file_path: Output AQL file path- **options: Additional export options-- Example:- >>> entities = [- ... {"id": "e1", "type": "Person", "name": "Alice"},- ... {"id": "e2", "type": "Organization", "name": "Acme Corp"}- ... ]- >>> exporter.export_entities(entities, "entities.aql")- """- knowledge_graph = {"entities": entities, "relationships": []}- self.export(knowledge_graph, file_path, **options)-- def export_relationships(- self,- relationships: List[Dict[str, Any]],- file_path: Union[str, Path],- **options,- ) -> None:- """- Export relationships to ArangoDB AQL format (edges only).-- Args:- relationships: List of relationship dictionaries- file_path: Output AQL file path- **options: Additional export options-- Example:- >>> relationships = [- ... {"id": "r1", "source": "e1", "target": "e2", "type": "WORKS_FOR"}- ... ]- >>> exporter.export_relationships(relationships, "relationships.aql")- """- knowledge_graph = {"entities": [], "relationships": relationships}- self.export(knowledge_graph, file_path, **options)+"""+ArangoDB AQL Export Module++This module provides comprehensive AQL export capabilities for the Semantica framework,+enabling export to ArangoDB multi-model graph databases.++Key Features:+ - AQL format export for ArangoDB+ - INSERT statement generation for vertex and edge collections+ - Configurable collection names+ - Entity and relationship identifier preservation+ - Batch insert support for performance+ - Proper string escaping and special character handling++Example Usage:+ >>> from semantica.export import ArangoAQLExporter+ >>> exporter = ArangoAQLExporter()+ >>> exporter.export_knowledge_graph(kg, "output.aql")+ >>> # With custom collection names+ >>> exporter = ArangoAQLExporter(+ ... vertex_collection="nodes",+ ... edge_collection="links"+ ... )+ >>> exporter.export(kg, "graph.aql")+"""++import json+from pathlib import Path+from typing import Any, Dict, List, Optional, Union++from ..utils.helpers import ensure_directory+from ..utils.logging import get_logger+from ..utils.progress_tracker import get_progress_tracker+++class ArangoAQLExporter:+ """+ ArangoDB AQL exporter for knowledge graphs.++ This class provides comprehensive AQL export functionality for knowledge graphs,+ supporting export to ArangoDB multi-model databases via AQL INSERT statements.++ Features:+ - AQL INSERT statement generation for vertices and edges+ - Configurable collection names+ - Entity and relationship identifier preservation+ - Batch insert support for performance+ - Proper string escaping and special character handling+ - Support for nested properties via JSON serialization++ Example Usage:+ >>> exporter = ArangoAQLExporter(+ ... vertex_collection="entities",+ ... edge_collection="relationships"+ ... )+ >>> exporter.export_knowledge_graph(kg, "output.aql")+ """++ def __init__(+ self,+ vertex_collection: str = "vertices",+ edge_collection: str = "edges",+ batch_size: int = 1000,+ include_collection_creation: bool = True,+ config: Optional[Dict[str, Any]] = None,+ **kwargs,+ ):+ """+ Initialize ArangoDB AQL exporter.++ Sets up the exporter with collection names and batch processing options.++ Args:+ vertex_collection: Name of the vertex collection+ (default: "vertices")+ edge_collection: Name of the edge collection (default: "edges")+ batch_size: Batch size for INSERT operations (default: 1000)+ include_collection_creation: Whether to include collection+ creation statements (default: True)+ config: Optional configuration dictionary (merged with kwargs)+ **kwargs: Additional configuration options+ """+ self.logger = get_logger("arango_aql_exporter")+ self.config = config or {}+ self.config.update(kwargs)++ # Validate collection names+ self._validate_collection_name(vertex_collection, "vertex_collection")+ self._validate_collection_name(edge_collection, "edge_collection")++ # AQL export configuration+ self.vertex_collection = vertex_collection+ self.edge_collection = edge_collection+ self.batch_size = batch_size+ self.include_collection_creation = include_collection_creation++ # Initialize progress tracker+ self.progress_tracker = get_progress_tracker()++ self.logger.debug(+ f"ArangoDB AQL exporter initialized: "+ f"vertex_collection={vertex_collection}, "+ f"edge_collection={edge_collection}, batch_size={batch_size}"+ )++ def export(+ self, knowledge_graph: Dict[str, Any], file_path: Union[str, Path], **options+ ) -> None:+ """+ Export knowledge graph to ArangoDB AQL format.++ This method exports a knowledge graph to AQL INSERT statements that can+ be imported into ArangoDB multi-model databases.++ Args:+ knowledge_graph: Knowledge graph dictionary containing:+ - entities: List of entity dictionaries+ - relationships: List of relationship dictionaries+ - nodes: List of node dictionaries (optional, alternative to+ entities)+ - edges: List of edge dictionaries (optional, alternative to+ relationships)+ file_path: Output AQL file path+ **options: Additional export options:+ - vertex_collection: Override default vertex collection name+ - edge_collection: Override default edge collection name++ Example:+ >>> kg = {+ ... "entities": [...],+ ... "relationships": [...]+ ... }+ >>> exporter.export(kg, "graph.aql")+ """+ file_path = Path(file_path)+ ensure_directory(file_path.parent)++ tracking_id = self.progress_tracker.start_tracking(+ file=str(file_path),+ module="export",+ submodule="ArangoAQLExporter",+ message=f"Exporting knowledge graph to ArangoDB AQL format: {file_path}",+ )++ try:+ # Override collection names if provided in options+ vertex_collection = options.pop("vertex_collection", self.vertex_collection)+ edge_collection = options.pop("edge_collection", self.edge_collection)++ # Validate overridden collection names+ if vertex_collection != self.vertex_collection:+ self._validate_collection_name(vertex_collection, "vertex_collection")+ if edge_collection != self.edge_collection:+ self._validate_collection_name(edge_collection, "edge_collection")++ # Generate AQL statements+ aql_statements = self._generate_aql_statements(+ knowledge_graph, vertex_collection, edge_collection, **options+ )++ # Write to file+ with open(file_path, "w", encoding="utf-8") as f:+ f.write("\n".join(aql_statements))++ self.progress_tracker.stop_tracking(+ tracking_id,+ status="completed",+ message=f"Exported {len(aql_statements)} AQL statements",+ )+ self.logger.info(+ f"Exported knowledge graph to ArangoDB AQL format: " f"{file_path}"+ )++ except Exception as e:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message=str(e)+ )+ raise++ def _generate_aql_statements(+ self,+ knowledge_graph: Dict[str, Any],+ vertex_collection: str,+ edge_collection: str,+ **options,+ ) -> List[str]:+ """+ Generate AQL INSERT statements from knowledge graph.++ Args:+ knowledge_graph: Knowledge graph dictionary+ vertex_collection: Vertex collection name+ edge_collection: Edge collection name+ **options: Additional options++ Returns:+ List of AQL statement strings+ """+ statements = []++ # Add collection creation statements if requested+ if self.include_collection_creation:+ statements.extend(+ self._generate_collection_creation(vertex_collection, edge_collection)+ )++ # Extract entities and relationships+ entities = knowledge_graph.get("entities", [])+ relationships = knowledge_graph.get("relationships", [])+ nodes = knowledge_graph.get("nodes", entities)+ edges = knowledge_graph.get("edges", relationships)++ # Use nodes/edges if entities/relationships are empty+ if not entities and nodes:+ entities = nodes+ if not relationships and edges:+ relationships = edges++ # Generate vertex INSERT statements+ vertex_statements = self._generate_vertex_inserts(entities, vertex_collection)+ statements.extend(vertex_statements)++ # Generate edge INSERT statements+ edge_statements = self._generate_edge_inserts(+ relationships, edge_collection, vertex_collection+ )+ statements.extend(edge_statements)++ return statements++ def _generate_collection_creation(+ self, vertex_collection: str, edge_collection: str+ ) -> List[str]:+ """+ Generate AQL collection creation statements.++ Args:+ vertex_collection: Vertex collection name+ edge_collection: Edge collection name++ Returns:+ List of collection creation statements+ """+ statements = [+ "// Create vertex collection if it doesn't exist",+ f"// db._createDocumentCollection('{vertex_collection}');",+ "",+ "// Create edge collection if it doesn't exist",+ f"// db._createEdgeCollection('{edge_collection}');",+ "",+ ]+ return statements++ def _generate_vertex_inserts(+ self, vertices: List[Dict[str, Any]], collection: str+ ) -> List[str]:+ """+ Generate AQL INSERT statements for vertices.++ Args:+ vertices: List of vertex/entity dictionaries+ collection: Vertex collection name++ Returns:+ List of AQL INSERT statements+ """+ statements = []++ # Add header comment+ statements.append(f"// Inserting {len(vertices)} vertices into {collection}")+ statements.append("")++ # Process vertices in batches+ for i in range(0, len(vertices), self.batch_size):+ batch = vertices[i : i + self.batch_size]+ batch_statement = self._create_vertex_batch_insert(batch, collection)+ statements.append(batch_statement)+ statements.append("")++ return statements++ def _create_vertex_batch_insert(+ self, vertices: List[Dict[str, Any]], collection: str+ ) -> str:+ """+ Create a batch INSERT statement for vertices.++ Args:+ vertices: Batch of vertex dictionaries+ collection: Vertex collection name++ Returns:+ AQL INSERT statement+ """+ if not vertices:+ return ""++ # Build document list+ documents = []+ for idx, vertex in enumerate(vertices):+ doc = self._convert_vertex_to_document(vertex, idx)+ documents.append(doc)++ # Format as AQL+ docs_json = json.dumps(documents, indent=2, ensure_ascii=False)+ statement = f"FOR doc IN {docs_json}\n INSERT doc INTO {collection}"++ return statement++ def _convert_vertex_to_document(+ self, vertex: Dict[str, Any], idx: int+ ) -> Dict[str, Any]:+ """+ Convert a vertex/entity to an ArangoDB document.++ Additional properties from the input entity are preserved as-is in the+ document root (not flattened or merged). Nested dictionaries and lists+ are preserved as JSON-serializable structures. The 'properties' field,+ if present, is also preserved as-is rather than being flattened into+ the document root.++ Args:+ vertex: Vertex/entity dictionary+ idx: Index for generating fallback IDs++ Returns:+ ArangoDB document dictionary+ """+ document = {}++ # Set _key from id or generate one+ vertex_id = vertex.get("id") or vertex.get("entity_id") or f"vertex_{idx}"+ document["_key"] = self._sanitize_key(str(vertex_id))++ # Add original ID if different from _key+ if str(vertex_id) != document["_key"]:+ document["original_id"] = str(vertex_id)++ # Add type/label information+ vertex_type = vertex.get("type") or vertex.get("entity_type", "Entity")+ document["type"] = vertex_type++ # Add name/label+ label = (+ vertex.get("label")+ or vertex.get("name")+ or vertex.get("text")+ or document["_key"]+ )+ document["name"] = label++ # Add all other properties+ for key, value in vertex.items():+ if key not in [+ "_key",+ "_id",+ "_rev",+ "id",+ "entity_id",+ "type",+ "entity_type",+ "label",+ "name",+ "text",+ ]:+ # Handle nested dictionaries and lists+ if isinstance(value, (dict, list)):+ document[key] = value+ elif value is not None:+ document[key] = value++ return document++ def _generate_edge_inserts(+ self, edges: List[Dict[str, Any]], collection: str, vertex_collection: str+ ) -> List[str]:+ """+ Generate AQL INSERT statements for edges.++ Args:+ edges: List of edge/relationship dictionaries+ collection: Edge collection name+ vertex_collection: Vertex collection name for _from/_to references++ Returns:+ List of AQL INSERT statements+ """+ statements = []++ # Add header comment+ statements.append(+ f"// Attempting to insert {len(edges)} edges into {collection}"+ )+ statements.append("")++ # Process edges in batches+ for i in range(0, len(edges), self.batch_size):+ batch = edges[i : i + self.batch_size]+ batch_statement = self._create_edge_batch_insert(+ batch, collection, vertex_collection+ )+ if batch_statement: # Only add non-empty statements+ statements.append(batch_statement)+ statements.append("")++ return statements++ def _create_edge_batch_insert(+ self, edges: List[Dict[str, Any]], collection: str, vertex_collection: str+ ) -> str:+ """+ Create a batch INSERT statement for edges.++ Args:+ edges: Batch of edge dictionaries+ collection: Edge collection name+ vertex_collection: Vertex collection name for _from/_to references++ Returns:+ AQL INSERT statement+ """+ # Build document list+ documents = []+ for idx, edge in enumerate(edges):+ doc = self._convert_edge_to_document(edge, idx, vertex_collection)+ if doc: # Only add valid edges+ documents.append(doc)++ if not documents:+ return ""++ # Format as AQL+ docs_json = json.dumps(documents, indent=2, ensure_ascii=False)+ statement = f"FOR doc IN {docs_json}\n INSERT doc INTO {collection}"++ return statement++ def _convert_edge_to_document(+ self, edge: Dict[str, Any], idx: int, vertex_collection: str+ ) -> Optional[Dict[str, Any]]:+ """+ Convert an edge/relationship to an ArangoDB edge document.++ Args:+ edge: Edge/relationship dictionary+ idx: Index for generating fallback IDs+ vertex_collection: Vertex collection name for _from/_to references++ Returns:+ ArangoDB edge document dictionary, or None if source/target missing+ """+ # Extract source and target+ source_id = edge.get("source") or edge.get("source_id")+ target_id = edge.get("target") or edge.get("target_id")++ if not source_id or not target_id:+ self.logger.warning(+ f"Skipping edge {idx}: missing source or target "+ f"(source={source_id}, target={target_id})"+ )+ return None++ document = {}++ # Set _key from id or generate one+ edge_id = edge.get("id") or edge.get("relationship_id") or f"edge_{idx}"+ document["_key"] = self._sanitize_key(str(edge_id))++ # Set _from and _to (required for edges in ArangoDB)+ document["_from"] = f"{vertex_collection}/{self._sanitize_key(str(source_id))}"+ document["_to"] = f"{vertex_collection}/{self._sanitize_key(str(target_id))}"++ # Add original ID if different from _key+ if str(edge_id) != document["_key"]:+ document["original_id"] = str(edge_id)++ # Add relationship type+ rel_type = edge.get("type") or edge.get("relationship_type", "RELATED_TO")+ document["type"] = rel_type++ # Add all other properties+ for key, value in edge.items():+ if key not in [+ "_key",+ "_id",+ "_rev",+ "_from",+ "_to",+ "id",+ "relationship_id",+ "source",+ "source_id",+ "target",+ "target_id",+ "type",+ "relationship_type",+ ]:+ # Handle nested dictionaries and lists+ if isinstance(value, (dict, list)):+ document[key] = value+ elif value is not None:+ document[key] = value++ return document++ def _validate_collection_name(self, name: str, param_name: str) -> None:+ """+ Validate an ArangoDB collection name.++ ArangoDB collection names must:+ - Start with a letter or underscore+ - Contain only alphanumeric characters, hyphens, and underscores+ - Not exceed 256 characters++ Args:+ name: Collection name to validate+ param_name: Parameter name for error messages++ Raises:+ ValueError: If the collection name is invalid+ """+ if not name:+ raise ValueError(f"{param_name} cannot be empty")++ if len(name) > 256:+ raise ValueError(+ f"{param_name} '{name}' exceeds maximum length of 256 characters"+ )++ # Check first character+ if not (name[0].isalpha() or name[0] == "_"):+ raise ValueError(+ f"{param_name} '{name}' must start with a letter or underscore"+ )++ # Check remaining characters+ for char in name:+ if not (char.isalnum() or char in ("-", "_")):+ raise ValueError(+ f"{param_name} '{name}' contains invalid character "+ f"'{char}'. Only alphanumeric characters, hyphens, and "+ "underscores are allowed."+ )++ def _sanitize_key(self, key: str) -> str:+ """+ Sanitize a key for use as ArangoDB _key.++ ArangoDB _key must contain only alphanumeric characters, hyphens,+ and underscores. It cannot start with an underscore (unless it's+ a system collection).++ Args:+ key: Original key string++ Returns:+ Sanitized key string+ """+ # Replace invalid characters with underscores+ sanitized = ""+ for char in key:+ if char.isalnum() or char in ("-", "_"):+ sanitized += char+ else:+ sanitized += "_"++ # Ensure key doesn't start with underscore+ if sanitized.startswith("_"):+ sanitized = "k" + sanitized++ # Ensure key is not empty+ if not sanitized:+ sanitized = "key"++ return sanitized++ def export_knowledge_graph(+ self, knowledge_graph: Dict[str, Any], file_path: Union[str, Path], **options+ ) -> None:+ """+ Export knowledge graph to ArangoDB AQL format.++ Convenience method that calls export().++ Args:+ knowledge_graph: Knowledge graph dictionary+ file_path: Output AQL file path+ **options: Additional export options++ Example:+ >>> kg = {+ ... "entities": [...],+ ... "relationships": [...]+ ... }+ >>> exporter.export_knowledge_graph(kg, "output.aql")+ """+ self.export(knowledge_graph, file_path, **options)++ def export_entities(+ self, entities: List[Dict[str, Any]], file_path: Union[str, Path], **options+ ) -> None:+ """+ Export entities to ArangoDB AQL format (vertices only).++ Args:+ entities: List of entity dictionaries+ file_path: Output AQL file path+ **options: Additional export options++ Example:+ >>> entities = [+ ... {"id": "e1", "type": "Person", "name": "Alice"},+ ... {"id": "e2", "type": "Organization", "name": "Acme Corp"}+ ... ]+ >>> exporter.export_entities(entities, "entities.aql")+ """+ knowledge_graph = {"entities": entities, "relationships": []}+ self.export(knowledge_graph, file_path, **options)++ def export_relationships(+ self,+ relationships: List[Dict[str, Any]],+ file_path: Union[str, Path],+ **options,+ ) -> None:+ """+ Export relationships to ArangoDB AQL format (edges only).++ Args:+ relationships: List of relationship dictionaries+ file_path: Output AQL file path+ **options: Additional export options++ Example:+ >>> relationships = [+ ... {"id": "r1", "source": "e1", "target": "e2", "type": "WORKS_FOR"}+ ... ]+ >>> exporter.export_relationships(relationships, "relationships.aql")+ """+ knowledge_graph = {"entities": [], "relationships": relationships}+ self.export(knowledge_graph, file_path, **options)diff --git a/semantica/export/parquet_exporter.py b/semantica/export/parquet_exporter.pyindex b2d6911e..8934f2e0 100644--- a/semantica/export/parquet_exporter.py+++ b/semantica/export/parquet_exporter.py@@ -1,700 +1,700 @@-"""-Apache Parquet Exporter Module--This module provides comprehensive Apache Parquet export capabilities for the-Semantica framework, enabling efficient columnar data export for entities,-relationships, and knowledge graphs optimized for analytics and data warehousing.--Key Features:- - Parquet file export (.parquet)- - Explicit schema definition (no inference)- - Entity and relationship export with metadata- - Knowledge graph export to multiple Parquet files- - Compatible with pandas, Spark, Snowflake, BigQuery, and Databricks- - Configurable compression (snappy, gzip, brotli, zstd, lz4)- - Batch export processing- - Structured metadata handling--Example Usage:- >>> from semantica.export import ParquetExporter- >>> exporter = ParquetExporter(compression="snappy")- >>> exporter.export_entities(entities, "entities.parquet")- >>> exporter.export_knowledge_graph(kg, "kg_base")--Author: Semantica Contributors-License: MIT-"""--import json-from pathlib import Path-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union--if TYPE_CHECKING:- import pyarrow as pa # noqa: F401--try:- import pyarrow as pa # noqa: F811- import pyarrow.parquet as pq-- PARQUET_AVAILABLE = True-- # Explicit Parquet Schemas (no inference)- ENTITY_SCHEMA = pa.schema(- [- pa.field("id", pa.string(), nullable=False),- pa.field("text", pa.string(), nullable=True),- pa.field("type", pa.string(), nullable=True),- pa.field("confidence", pa.float64(), nullable=True),- pa.field("start", pa.int64(), nullable=True),- pa.field("end", pa.int64(), nullable=True),- pa.field(- "metadata",- pa.struct(- [- pa.field("keys", pa.list_(pa.string())),- pa.field("values", pa.list_(pa.string())),- ]- ),- nullable=True,- ),- ]- )-- RELATIONSHIP_SCHEMA = pa.schema(- [- pa.field("id", pa.string(), nullable=False),- pa.field("source_id", pa.string(), nullable=False),- pa.field("target_id", pa.string(), nullable=False),- pa.field("type", pa.string(), nullable=True),- pa.field("confidence", pa.float64(), nullable=True),- pa.field(- "metadata",- pa.struct(- [- pa.field("keys", pa.list_(pa.string())),- pa.field("values", pa.list_(pa.string())),- ]- ),- nullable=True,- ),- ]- )-except ImportError:- PARQUET_AVAILABLE = False- ENTITY_SCHEMA = None- RELATIONSHIP_SCHEMA = None--from ..utils.exceptions import ProcessingError, ValidationError-from ..utils.helpers import ensure_directory-from ..utils.logging import get_logger-from ..utils.progress_tracker import get_progress_tracker---class ParquetExporter:- """- Apache Parquet exporter for knowledge graphs and structured data.-- This class provides comprehensive Parquet export functionality for entities,- relationships, and knowledge graphs. Uses explicit schemas for type safety- and compatibility with analytics platforms like pandas, Spark, Snowflake,- BigQuery, and Databricks.-- Features:- - Entity and relationship export- - Knowledge graph export to multiple Parquet files- - Explicit schema definition (no inference)- - Metadata serialization as Parquet struct fields- - Configurable compression (snappy, gzip, brotli, zstd, lz4)- - Compatible with major analytics platforms- - Progress tracking and error handling-- Example Usage:- >>> exporter = ParquetExporter(compression="snappy")- >>> exporter.export_entities(entities, "entities.parquet")- >>> exporter.export_knowledge_graph(kg, "output_base")- """-- def __init__(- self,- compression: str = "snappy",- config: Optional[Dict[str, Any]] = None,- **kwargs,- ):- """- Initialize Parquet exporter.-- Sets up the exporter with specified Parquet formatting options.-- Args:- compression: Compression codec (default: "snappy")- - "snappy": Snappy compression (fast, good compression)- - "gzip": GZIP compression (slower, better compression)- - "brotli": Brotli compression (slow, best compression)- - "zstd": Zstandard compression (balanced)- - "lz4": LZ4 compression (very fast, moderate compression)- - "none" or None: No compression- config: Optional configuration dictionary (merged with kwargs)- **kwargs: Additional configuration options-- Raises:- ImportError: If pyarrow is not installed- """- if not PARQUET_AVAILABLE:- raise ImportError(- "pyarrow is not installed. Please install it with: "- "pip install pyarrow"- )-- self.logger = get_logger("parquet_exporter")- self.config = config or {}- self.config.update(kwargs)-- # Parquet configuration- self.compression = compression if compression != "none" else None-- # Initialize progress tracker- self.progress_tracker = get_progress_tracker()- # Ensure progress tracker is enabled- if not self.progress_tracker.enabled:- self.progress_tracker.enabled = True-- self.logger.debug(f"Parquet exporter initialized: compression={compression}")-- def export(- self,- data: Union[List[Dict[str, Any]], Dict[str, Any]],- file_path: Union[str, Path],- schema: Optional["pa.Schema"] = None,- **options,- ) -> None:- """- Export data to Parquet file(s).-- This method handles both single Parquet file export (from list) and multiple- Parquet file export (from dictionary with multiple keys).-- Args:- data: Data to export:- - List of dicts: Exports to single Parquet file- - Dict with list values: Exports each key as separate Parquet file- file_path: Output file path (base path for dict exports)- schema: Parquet schema to use (default: auto-select based on data)- **options: Additional options-- Raises:- ValidationError: If data type is unsupported-- Example:- >>> # Single Parquet file- >>> exporter.export([{"id": "1", "name": "A"}], "data.parquet")- >>> # Multiple Parquet files- >>> exporter.export(- ... {"entities": [...], "relationships": [...]},- ... "output_base"- ... )- """- # Track Parquet export- tracking_id = self.progress_tracker.start_tracking(- file=str(file_path),- module="export",- submodule="ParquetExporter",- message=f"Exporting data to Parquet: {file_path}",- )-- try:- file_path = Path(file_path)- ensure_directory(file_path.parent)-- self.logger.debug(f"Exporting data to Parquet: {file_path}")-- # Handle different data structures- if isinstance(data, dict):- # Export each key as separate Parquet file- exported_files = []- self.progress_tracker.update_tracking(- tracking_id, message=f"Exporting {len(data)} data groups..."- )- for key, value in data.items():- if isinstance(value, list):- output_path = (- file_path.parent / f"{file_path.stem}_{key}.parquet"- )-- # Use dedicated export methods for entities and relationships- # to ensure proper normalization- if key == "entities" and schema is None:- self.export_entities(value, output_path, **options)- elif key == "relationships" and schema is None:- self.export_relationships(value, output_path, **options)- else:- # For other keys, write directly with provided schema- self._write_parquet(- value, output_path, schema=schema, **options- )-- exported_files.append(output_path)- else:- self.logger.warning(- f"Skipping key '{key}': value is not a list "- f"(type: {type(value)})"- )-- self.logger.info(- f"Exported {len(exported_files)} Parquet file(s) from dictionary: "- f"{', '.join(str(f) for f in exported_files)}"- )- self.progress_tracker.stop_tracking(- tracking_id,- status="completed",- message=f"Exported {len(exported_files)} Parquet files",- )- elif isinstance(data, list):- # Single Parquet file - auto-detect if entities or relationships- self.progress_tracker.update_tracking(- tracking_id, message=f"Exporting {len(data)} records..."- )-- # If no schema provided, try to auto-detect from data structure- if schema is None:- if not data:- raise ValidationError(- "Cannot export empty list without explicit schema. "- "Provide a schema or use "- "export_entities/export_relationships."- )- sample = data[0]- has_source = any(- k in sample for k in ["source_id", "source", "from_id", "from"]- )- has_target = any(- k in sample for k in ["target_id", "target", "to_id", "to"]- )-- if has_source and has_target:- # Use dedicated method for relationship normalization- self.export_relationships(data, file_path, **options)- else:- # Use dedicated method for entity normalization- self.export_entities(data, file_path, **options)- else:- # Schema provided - write directly- self._write_parquet(data, file_path, schema=schema, **options)-- self.logger.info(f"Exported Parquet to: {file_path}")- self.progress_tracker.stop_tracking(- tracking_id,- status="completed",- message=f"Exported Parquet to: {file_path}",- )- else:- raise ValidationError(- f"Unsupported data type: {type(data)}. "- "Expected list of dicts or dict with list values."- )-- except Exception as e:- self.progress_tracker.stop_tracking(- tracking_id, status="failed", message=str(e)- )- raise-- def export_entities(- self, entities: List[Dict[str, Any]], file_path: Union[str, Path], **options- ) -> None:- """- Export entities to Parquet file.-- This method normalizes entity data to a consistent format and exports- to Parquet using the explicit ENTITY_SCHEMA. Handles various entity field- name variations and serializes metadata as Parquet structs.-- Normalized Fields:- - id: Entity identifier (required)- - text: Entity text/label/name- - type: Entity type- - confidence: Confidence score- - start: Start offset/position- - end: End offset/position- - metadata: Metadata as struct (keys and values lists)-- Args:- entities: List of entity dictionaries with various field names- file_path: Output Parquet file path- **options: Additional options passed to _write_parquet()-- Raises:- ValidationError: If entities list is empty-- Example:- >>> entities = [- ... {"id": "e1", "text": "Entity 1", "type": "PERSON"},- ... {"id": "e2", "label": "Entity 2", "entity_type": "ORG"}- ... ]- >>> exporter.export_entities(entities, "entities.parquet")- """- if not entities:- raise ValidationError("No entities to export. Entities list is empty.")-- self.logger.debug(f"Exporting {len(entities)} entity(ies) to Parquet")-- # Normalize entity data to consistent format- normalized_entities = []- for i, entity in enumerate(entities):- if not isinstance(entity, dict):- self.logger.warning(f"Entity {i} is not a dictionary, skipping")- continue-- # Extract and normalize fields- entity_id = entity.get("id") or entity.get("entity_id")- if not entity_id:- self.logger.warning(f"Entity {i} missing ID, skipping")- continue-- # Normalize confidence to float, with validation- raw_confidence = entity.get("confidence")- confidence_value = None- if raw_confidence is not None:- try:- confidence_value = float(raw_confidence)- except (TypeError, ValueError):- self.logger.warning(- f"Entity {i} has non-numeric confidence {raw_confidence!r}; "- "setting to None"- )- confidence_value = None-- # Normalize start/end to int, with validation- start_value = entity.get("start")- if start_value is None:- start_value = entity.get("start_offset")- if start_value is not None:- try:- start_value = int(start_value)- except (TypeError, ValueError):- self.logger.warning(- f"Entity {i} has non-integer start {start_value!r}; "- "setting to None"- )- start_value = None-- end_value = entity.get("end")- if end_value is None:- end_value = entity.get("end_offset")- if end_value is not None:- try:- end_value = int(end_value)- except (TypeError, ValueError):- self.logger.warning(- f"Entity {i} has non-integer end {end_value!r}; "- "setting to None"- )- end_value = None-- normalized = {- "id": str(entity_id),- "text": (- entity.get("text") or entity.get("label") or entity.get("name")- ),- "type": entity.get("type") or entity.get("entity_type"),- "confidence": confidence_value,- "start": start_value,- "end": end_value,- }-- # Convert metadata to struct format (keys and values lists)- if "metadata" in entity and isinstance(entity["metadata"], dict):- metadata_dict = entity["metadata"]- normalized["metadata"] = {- "keys": list(metadata_dict.keys()),- "values": [- json.dumps(v) if not isinstance(v, str) else v- for v in metadata_dict.values()- ],- }- else:- normalized["metadata"] = None-- normalized_entities.append(normalized)-- self.logger.debug(- f"Normalized {len(normalized_entities)} entity(ies) for Parquet export"- )-- if not normalized_entities:- raise ValidationError(- "No valid entities to export after normalization. "- "All entities were skipped due to missing IDs or invalid format."- )-- self._write_parquet(- normalized_entities, file_path, schema=ENTITY_SCHEMA, **options- )-- def export_relationships(- self,- relationships: List[Dict[str, Any]],- file_path: Union[str, Path],- **options,- ) -> None:- """- Export relationships to Parquet file.-- This method normalizes relationship data to a consistent format and exports- to Parquet using the explicit RELATIONSHIP_SCHEMA. Handles various relationship- field name variations and serializes metadata as Parquet structs.-- Normalized Fields:- - id: Relationship identifier (generated if missing)- - source_id: Source entity ID (required)- - target_id: Target entity ID (required)- - type: Relationship type- - confidence: Confidence score- - metadata: Metadata as struct (keys and values lists)-- Args:- relationships: List of relationship dictionaries with various field names- file_path: Output Parquet file path- **options: Additional options passed to _write_parquet()-- Raises:- ValidationError: If relationships list is empty-- Example:- >>> relationships = [- ... {"id": "r1", "source": "e1", "target": "e2", "type": "KNOWS"},- ... {"source_id": "e2", "target_id": "e3", "relationship_type": "LIKES"}- ... ]- >>> exporter.export_relationships(relationships, "relationships.parquet")- """- if not relationships:- raise ValidationError(- "No relationships to export. Relationships list is empty."- )-- self.logger.debug(f"Exporting {len(relationships)} relationship(s) to Parquet")-- # Normalize relationship data to consistent format- normalized_rels = []- for i, rel in enumerate(relationships):- if not isinstance(rel, dict):- self.logger.warning(f"Relationship {i} is not a dictionary, skipping")- continue-- # Extract source and target IDs- source_id = (- rel.get("source_id")- or rel.get("source")- or rel.get("from_id")- or rel.get("from")- )- target_id = (- rel.get("target_id")- or rel.get("target")- or rel.get("to_id")- or rel.get("to")- )-- if not source_id or not target_id:- self.logger.warning(- f"Relationship {i} missing source or target ID, skipping"- )- continue-- # Generate ID if missing- rel_id = rel.get("id") or rel.get("relationship_id") or f"rel_{i}"-- # Normalize confidence to float, with validation- raw_confidence = rel.get("confidence")- confidence_value = None- if raw_confidence is not None:- try:- confidence_value = float(raw_confidence)- except (TypeError, ValueError):- self.logger.warning(- f"Relationship {i} has non-numeric confidence "- f"{raw_confidence!r}; setting to None"- )- confidence_value = None-- normalized = {- "id": str(rel_id),- "source_id": str(source_id),- "target_id": str(target_id),- "type": (- rel.get("type")- or rel.get("relationship_type")- or rel.get("relation_type")- ),- "confidence": confidence_value,- }-- # Convert metadata to struct format (keys and values lists)- if "metadata" in rel and isinstance(rel["metadata"], dict):- metadata_dict = rel["metadata"]- normalized["metadata"] = {- "keys": list(metadata_dict.keys()),- "values": [- json.dumps(v) if not isinstance(v, str) else v- for v in metadata_dict.values()- ],- }- else:- normalized["metadata"] = None-- normalized_rels.append(normalized)-- self.logger.debug(- f"Normalized {len(normalized_rels)} relationship(s) for Parquet export"- )-- if not normalized_rels:- raise ValidationError(- "No valid relationships to export after normalization. "- "Ensure each relationship is a dictionary and includes valid "- "'source'/'source_id' and 'target'/'target_id' fields."- )-- self._write_parquet(- normalized_rels, file_path, schema=RELATIONSHIP_SCHEMA, **options- )-- def export_knowledge_graph(- self, kg: Dict[str, Any], base_path: Union[str, Path], **options- ) -> None:- """- Export knowledge graph to multiple Parquet files.-- This method exports a knowledge graph to separate Parquet files for- entities and relationships. Files are named using the base_path with- suffixes: _entities.parquet and _relationships.parquet.-- Args:- kg: Knowledge graph dictionary with 'entities' and 'relationships' keys- base_path: Base path for output files (without extension)- **options: Additional options passed to export methods-- Raises:- ValidationError: If knowledge graph is missing required keys-- Example:- >>> kg = {- ... "entities": [...],- ... "relationships": [...]- ... }- >>> exporter.export_knowledge_graph(kg, "output/kg_base")- # Creates: output/kg_base_entities.parquet and- # output/kg_base_relationships.parquet- """- if not isinstance(kg, dict):- raise ValidationError(- f"Knowledge graph must be a dictionary, got {type(kg)}"- )-- if "entities" not in kg and "relationships" not in kg:- raise ValidationError(- "Knowledge graph must contain 'entities' or 'relationships' key"- )-- base_path = Path(base_path)- ensure_directory(base_path.parent)-- self.logger.debug(f"Exporting knowledge graph to Parquet: {base_path}")-- # Track KG export- tracking_id = self.progress_tracker.start_tracking(- file=str(base_path),- module="export",- submodule="ParquetExporter",- message=f"Exporting knowledge graph to Parquet: {base_path}",- )-- try:- exported_files = []-- # Export entities- if "entities" in kg and kg["entities"]:- entities_path = base_path.parent / f"{base_path.stem}_entities.parquet"- self.export_entities(kg["entities"], entities_path, **options)- exported_files.append(entities_path)- self.logger.info(f"Exported entities to: {entities_path}")-- # Export relationships- if "relationships" in kg and kg["relationships"]:- rels_path = base_path.parent / f"{base_path.stem}_relationships.parquet"- self.export_relationships(kg["relationships"], rels_path, **options)- exported_files.append(rels_path)- self.logger.info(f"Exported relationships to: {rels_path}")-- self.logger.info(- f"Exported knowledge graph to {len(exported_files)} Parquet file(s)"- )- self.progress_tracker.stop_tracking(- tracking_id,- status="completed",- message=f"Exported {len(exported_files)} Parquet files",- )-- except Exception as e:- self.progress_tracker.stop_tracking(- tracking_id, status="failed", message=str(e)- )- raise-- def _write_parquet(- self,- data: List[Dict[str, Any]],- file_path: Union[str, Path],- schema: Optional["pa.Schema"] = None,- **options,- ) -> None:- """- Write data to Parquet file.-- Internal method that handles the actual Parquet file writing using pyarrow.-- Args:- data: List of dictionaries to write- file_path: Output Parquet file path- schema: Parquet schema to use (required for type safety)- **options: Additional parquet write options-- Raises:- ProcessingError: If Parquet writing fails- ValidationError: If schema is not provided- """- if not schema:- raise ValidationError("Schema is required for Parquet export")-- # Ensure file_path is a Path object- file_path = Path(file_path)-- if not data:- self.logger.warning(f"No data to write to {file_path}")- # Write empty Parquet file with schema- empty_table = pa.table({field.name: [] for field in schema}, schema=schema)- pq.write_table(- empty_table, str(file_path), compression=self.compression, **options- )- return-- try:- # Create PyArrow table from data using explicit schema- table = pa.Table.from_pylist(data, schema=schema)-- # Write to Parquet file- pq.write_table(- table, str(file_path), compression=self.compression, **options- )-- file_size = file_path.stat().st_size- self.logger.debug(- f"Wrote {len(data)} row(s) to {file_path} ({file_size} bytes)"- )-- except pa.ArrowInvalid as e:- raise ProcessingError(- f"Failed to create Parquet table: {e}. "- "Check that data matches schema."- )- except Exception as e:- raise ProcessingError(f"Failed to write Parquet file: {e}")+"""+Apache Parquet Exporter Module++This module provides comprehensive Apache Parquet export capabilities for the+Semantica framework, enabling efficient columnar data export for entities,+relationships, and knowledge graphs optimized for analytics and data warehousing.++Key Features:+ - Parquet file export (.parquet)+ - Explicit schema definition (no inference)+ - Entity and relationship export with metadata+ - Knowledge graph export to multiple Parquet files+ - Compatible with pandas, Spark, Snowflake, BigQuery, and Databricks+ - Configurable compression (snappy, gzip, brotli, zstd, lz4)+ - Batch export processing+ - Structured metadata handling++Example Usage:+ >>> from semantica.export import ParquetExporter+ >>> exporter = ParquetExporter(compression="snappy")+ >>> exporter.export_entities(entities, "entities.parquet")+ >>> exporter.export_knowledge_graph(kg, "kg_base")++Author: Semantica Contributors+License: MIT+"""++import json+from pathlib import Path+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union++if TYPE_CHECKING:+ import pyarrow as pa # noqa: F401++try:+ import pyarrow as pa # noqa: F811+ import pyarrow.parquet as pq++ PARQUET_AVAILABLE = True++ # Explicit Parquet Schemas (no inference)+ ENTITY_SCHEMA = pa.schema(+ [+ pa.field("id", pa.string(), nullable=False),+ pa.field("text", pa.string(), nullable=True),+ pa.field("type", pa.string(), nullable=True),+ pa.field("confidence", pa.float64(), nullable=True),+ pa.field("start", pa.int64(), nullable=True),+ pa.field("end", pa.int64(), nullable=True),+ pa.field(+ "metadata",+ pa.struct(+ [+ pa.field("keys", pa.list_(pa.string())),+ pa.field("values", pa.list_(pa.string())),+ ]+ ),+ nullable=True,+ ),+ ]+ )++ RELATIONSHIP_SCHEMA = pa.schema(+ [+ pa.field("id", pa.string(), nullable=False),+ pa.field("source_id", pa.string(), nullable=False),+ pa.field("target_id", pa.string(), nullable=False),+ pa.field("type", pa.string(), nullable=True),+ pa.field("confidence", pa.float64(), nullable=True),+ pa.field(+ "metadata",+ pa.struct(+ [+ pa.field("keys", pa.list_(pa.string())),+ pa.field("values", pa.list_(pa.string())),+ ]+ ),+ nullable=True,+ ),+ ]+ )+except ImportError:+ PARQUET_AVAILABLE = False+ ENTITY_SCHEMA = None+ RELATIONSHIP_SCHEMA = None++from ..utils.exceptions import ProcessingError, ValidationError+from ..utils.helpers import ensure_directory+from ..utils.logging import get_logger+from ..utils.progress_tracker import get_progress_tracker+++class ParquetExporter:+ """+ Apache Parquet exporter for knowledge graphs and structured data.++ This class provides comprehensive Parquet export functionality for entities,+ relationships, and knowledge graphs. Uses explicit schemas for type safety+ and compatibility with analytics platforms like pandas, Spark, Snowflake,+ BigQuery, and Databricks.++ Features:+ - Entity and relationship export+ - Knowledge graph export to multiple Parquet files+ - Explicit schema definition (no inference)+ - Metadata serialization as Parquet struct fields+ - Configurable compression (snappy, gzip, brotli, zstd, lz4)+ - Compatible with major analytics platforms+ - Progress tracking and error handling++ Example Usage:+ >>> exporter = ParquetExporter(compression="snappy")+ >>> exporter.export_entities(entities, "entities.parquet")+ >>> exporter.export_knowledge_graph(kg, "output_base")+ """++ def __init__(+ self,+ compression: str = "snappy",+ config: Optional[Dict[str, Any]] = None,+ **kwargs,+ ):+ """+ Initialize Parquet exporter.++ Sets up the exporter with specified Parquet formatting options.++ Args:+ compression: Compression codec (default: "snappy")+ - "snappy": Snappy compression (fast, good compression)+ - "gzip": GZIP compression (slower, better compression)+ - "brotli": Brotli compression (slow, best compression)+ - "zstd": Zstandard compression (balanced)+ - "lz4": LZ4 compression (very fast, moderate compression)+ - "none" or None: No compression+ config: Optional configuration dictionary (merged with kwargs)+ **kwargs: Additional configuration options++ Raises:+ ImportError: If pyarrow is not installed+ """+ if not PARQUET_AVAILABLE:+ raise ImportError(+ "pyarrow is not installed. Please install it with: "+ "pip install pyarrow"+ )++ self.logger = get_logger("parquet_exporter")+ self.config = config or {}+ self.config.update(kwargs)++ # Parquet configuration+ self.compression = compression if compression != "none" else None++ # Initialize progress tracker+ self.progress_tracker = get_progress_tracker()+ # Ensure progress tracker is enabled+ if not self.progress_tracker.enabled:+ self.progress_tracker.enabled = True++ self.logger.debug(f"Parquet exporter initialized: compression={compression}")++ def export(+ self,+ data: Union[List[Dict[str, Any]], Dict[str, Any]],+ file_path: Union[str, Path],+ schema: Optional["pa.Schema"] = None,+ **options,+ ) -> None:+ """+ Export data to Parquet file(s).++ This method handles both single Parquet file export (from list) and multiple+ Parquet file export (from dictionary with multiple keys).++ Args:+ data: Data to export:+ - List of dicts: Exports to single Parquet file+ - Dict with list values: Exports each key as separate Parquet file+ file_path: Output file path (base path for dict exports)+ schema: Parquet schema to use (default: auto-select based on data)+ **options: Additional options++ Raises:+ ValidationError: If data type is unsupported++ Example:+ >>> # Single Parquet file+ >>> exporter.export([{"id": "1", "name": "A"}], "data.parquet")+ >>> # Multiple Parquet files+ >>> exporter.export(+ ... {"entities": [...], "relationships": [...]},+ ... "output_base"+ ... )+ """+ # Track Parquet export+ tracking_id = self.progress_tracker.start_tracking(+ file=str(file_path),+ module="export",+ submodule="ParquetExporter",+ message=f"Exporting data to Parquet: {file_path}",+ )++ try:+ file_path = Path(file_path)+ ensure_directory(file_path.parent)++ self.logger.debug(f"Exporting data to Parquet: {file_path}")++ # Handle different data structures+ if isinstance(data, dict):+ # Export each key as separate Parquet file+ exported_files = []+ self.progress_tracker.update_tracking(+ tracking_id, message=f"Exporting {len(data)} data groups..."+ )+ for key, value in data.items():+ if isinstance(value, list):+ output_path = (+ file_path.parent / f"{file_path.stem}_{key}.parquet"+ )++ # Use dedicated export methods for entities and relationships+ # to ensure proper normalization+ if key == "entities" and schema is None:+ self.export_entities(value, output_path, **options)+ elif key == "relationships" and schema is None:+ self.export_relationships(value, output_path, **options)+ else:+ # For other keys, write directly with provided schema+ self._write_parquet(+ value, output_path, schema=schema, **options+ )++ exported_files.append(output_path)+ else:+ self.logger.warning(+ f"Skipping key '{key}': value is not a list "+ f"(type: {type(value)})"+ )++ self.logger.info(+ f"Exported {len(exported_files)} Parquet file(s) from dictionary: "+ f"{', '.join(str(f) for f in exported_files)}"+ )+ self.progress_tracker.stop_tracking(+ tracking_id,+ status="completed",+ message=f"Exported {len(exported_files)} Parquet files",+ )+ elif isinstance(data, list):+ # Single Parquet file - auto-detect if entities or relationships+ self.progress_tracker.update_tracking(+ tracking_id, message=f"Exporting {len(data)} records..."+ )++ # If no schema provided, try to auto-detect from data structure+ if schema is None:+ if not data:+ raise ValidationError(+ "Cannot export empty list without explicit schema. "+ "Provide a schema or use "+ "export_entities/export_relationships."+ )+ sample = data[0]+ has_source = any(+ k in sample for k in ["source_id", "source", "from_id", "from"]+ )+ has_target = any(+ k in sample for k in ["target_id", "target", "to_id", "to"]+ )++ if has_source and has_target:+ # Use dedicated method for relationship normalization+ self.export_relationships(data, file_path, **options)+ else:+ # Use dedicated method for entity normalization+ self.export_entities(data, file_path, **options)+ else:+ # Schema provided - write directly+ self._write_parquet(data, file_path, schema=schema, **options)++ self.logger.info(f"Exported Parquet to: {file_path}")+ self.progress_tracker.stop_tracking(+ tracking_id,+ status="completed",+ message=f"Exported Parquet to: {file_path}",+ )+ else:+ raise ValidationError(+ f"Unsupported data type: {type(data)}. "+ "Expected list of dicts or dict with list values."+ )++ except Exception as e:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message=str(e)+ )+ raise++ def export_entities(+ self, entities: List[Dict[str, Any]], file_path: Union[str, Path], **options+ ) -> None:+ """+ Export entities to Parquet file.++ This method normalizes entity data to a consistent format and exports+ to Parquet using the explicit ENTITY_SCHEMA. Handles various entity field+ name variations and serializes metadata as Parquet structs.++ Normalized Fields:+ - id: Entity identifier (required)+ - text: Entity text/label/name+ - type: Entity type+ - confidence: Confidence score+ - start: Start offset/position+ - end: End offset/position+ - metadata: Metadata as struct (keys and values lists)++ Args:+ entities: List of entity dictionaries with various field names+ file_path: Output Parquet file path+ **options: Additional options passed to _write_parquet()++ Raises:+ ValidationError: If entities list is empty++ Example:+ >>> entities = [+ ... {"id": "e1", "text": "Entity 1", "type": "PERSON"},+ ... {"id": "e2", "label": "Entity 2", "entity_type": "ORG"}+ ... ]+ >>> exporter.export_entities(entities, "entities.parquet")+ """+ if not entities:+ raise ValidationError("No entities to export. Entities list is empty.")++ self.logger.debug(f"Exporting {len(entities)} entity(ies) to Parquet")++ # Normalize entity data to consistent format+ normalized_entities = []+ for i, entity in enumerate(entities):+ if not isinstance(entity, dict):+ self.logger.warning(f"Entity {i} is not a dictionary, skipping")+ continue++ # Extract and normalize fields+ entity_id = entity.get("id") or entity.get("entity_id")+ if not entity_id:+ self.logger.warning(f"Entity {i} missing ID, skipping")+ continue++ # Normalize confidence to float, with validation+ raw_confidence = entity.get("confidence")+ confidence_value = None+ if raw_confidence is not None:+ try:+ confidence_value = float(raw_confidence)+ except (TypeError, ValueError):+ self.logger.warning(+ f"Entity {i} has non-numeric confidence {raw_confidence!r}; "+ "setting to None"+ )+ confidence_value = None++ # Normalize start/end to int, with validation+ start_value = entity.get("start")+ if start_value is None:+ start_value = entity.get("start_offset")+ if start_value is not None:+ try:+ start_value = int(start_value)+ except (TypeError, ValueError):+ self.logger.warning(+ f"Entity {i} has non-integer start {start_value!r}; "+ "setting to None"+ )+ start_value = None++ end_value = entity.get("end")+ if end_value is None:+ end_value = entity.get("end_offset")+ if end_value is not None:+ try:+ end_value = int(end_value)+ except (TypeError, ValueError):+ self.logger.warning(+ f"Entity {i} has non-integer end {end_value!r}; "+ "setting to None"+ )+ end_value = None++ normalized = {+ "id": str(entity_id),+ "text": (+ entity.get("text") or entity.get("label") or entity.get("name")+ ),+ "type": entity.get("type") or entity.get("entity_type"),+ "confidence": confidence_value,+ "start": start_value,+ "end": end_value,+ }++ # Convert metadata to struct format (keys and values lists)+ if "metadata" in entity and isinstance(entity["metadata"], dict):+ metadata_dict = entity["metadata"]+ normalized["metadata"] = {+ "keys": list(metadata_dict.keys()),+ "values": [+ json.dumps(v) if not isinstance(v, str) else v+ for v in metadata_dict.values()+ ],+ }+ else:+ normalized["metadata"] = None++ normalized_entities.append(normalized)++ self.logger.debug(+ f"Normalized {len(normalized_entities)} entity(ies) for Parquet export"+ )++ if not normalized_entities:+ raise ValidationError(+ "No valid entities to export after normalization. "+ "All entities were skipped due to missing IDs or invalid format."+ )++ self._write_parquet(+ normalized_entities, file_path, schema=ENTITY_SCHEMA, **options+ )++ def export_relationships(+ self,+ relationships: List[Dict[str, Any]],+ file_path: Union[str, Path],+ **options,+ ) -> None:+ """+ Export relationships to Parquet file.++ This method normalizes relationship data to a consistent format and exports+ to Parquet using the explicit RELATIONSHIP_SCHEMA. Handles various relationship+ field name variations and serializes metadata as Parquet structs.++ Normalized Fields:+ - id: Relationship identifier (generated if missing)+ - source_id: Source entity ID (required)+ - target_id: Target entity ID (required)+ - type: Relationship type+ - confidence: Confidence score+ - metadata: Metadata as struct (keys and values lists)++ Args:+ relationships: List of relationship dictionaries with various field names+ file_path: Output Parquet file path+ **options: Additional options passed to _write_parquet()++ Raises:+ ValidationError: If relationships list is empty++ Example:+ >>> relationships = [+ ... {"id": "r1", "source": "e1", "target": "e2", "type": "KNOWS"},+ ... {"source_id": "e2", "target_id": "e3", "relationship_type": "LIKES"}+ ... ]+ >>> exporter.export_relationships(relationships, "relationships.parquet")+ """+ if not relationships:+ raise ValidationError(+ "No relationships to export. Relationships list is empty."+ )++ self.logger.debug(f"Exporting {len(relationships)} relationship(s) to Parquet")++ # Normalize relationship data to consistent format+ normalized_rels = []+ for i, rel in enumerate(relationships):+ if not isinstance(rel, dict):+ self.logger.warning(f"Relationship {i} is not a dictionary, skipping")+ continue++ # Extract source and target IDs+ source_id = (+ rel.get("source_id")+ or rel.get("source")+ or rel.get("from_id")+ or rel.get("from")+ )+ target_id = (+ rel.get("target_id")+ or rel.get("target")+ or rel.get("to_id")+ or rel.get("to")+ )++ if not source_id or not target_id:+ self.logger.warning(+ f"Relationship {i} missing source or target ID, skipping"+ )+ continue++ # Generate ID if missing+ rel_id = rel.get("id") or rel.get("relationship_id") or f"rel_{i}"++ # Normalize confidence to float, with validation+ raw_confidence = rel.get("confidence")+ confidence_value = None+ if raw_confidence is not None:+ try:+ confidence_value = float(raw_confidence)+ except (TypeError, ValueError):+ self.logger.warning(+ f"Relationship {i} has non-numeric confidence "+ f"{raw_confidence!r}; setting to None"+ )+ confidence_value = None++ normalized = {+ "id": str(rel_id),+ "source_id": str(source_id),+ "target_id": str(target_id),+ "type": (+ rel.get("type")+ or rel.get("relationship_type")+ or rel.get("relation_type")+ ),+ "confidence": confidence_value,+ }++ # Convert metadata to struct format (keys and values lists)+ if "metadata" in rel and isinstance(rel["metadata"], dict):+ metadata_dict = rel["metadata"]+ normalized["metadata"] = {+ "keys": list(metadata_dict.keys()),+ "values": [+ json.dumps(v) if not isinstance(v, str) else v+ for v in metadata_dict.values()+ ],+ }+ else:+ normalized["metadata"] = None++ normalized_rels.append(normalized)++ self.logger.debug(+ f"Normalized {len(normalized_rels)} relationship(s) for Parquet export"+ )++ if not normalized_rels:+ raise ValidationError(+ "No valid relationships to export after normalization. "+ "Ensure each relationship is a dictionary and includes valid "+ "'source'/'source_id' and 'target'/'target_id' fields."+ )++ self._write_parquet(+ normalized_rels, file_path, schema=RELATIONSHIP_SCHEMA, **options+ )++ def export_knowledge_graph(+ self, kg: Dict[str, Any], base_path: Union[str, Path], **options+ ) -> None:+ """+ Export knowledge graph to multiple Parquet files.++ This method exports a knowledge graph to separate Parquet files for+ entities and relationships. Files are named using the base_path with+ suffixes: _entities.parquet and _relationships.parquet.++ Args:+ kg: Knowledge graph dictionary with 'entities' and 'relationships' keys+ base_path: Base path for output files (without extension)+ **options: Additional options passed to export methods++ Raises:+ ValidationError: If knowledge graph is missing required keys++ Example:+ >>> kg = {+ ... "entities": [...],+ ... "relationships": [...]+ ... }+ >>> exporter.export_knowledge_graph(kg, "output/kg_base")+ # Creates: output/kg_base_entities.parquet and+ # output/kg_base_relationships.parquet+ """+ if not isinstance(kg, dict):+ raise ValidationError(+ f"Knowledge graph must be a dictionary, got {type(kg)}"+ )++ if "entities" not in kg and "relationships" not in kg:+ raise ValidationError(+ "Knowledge graph must contain 'entities' or 'relationships' key"+ )++ base_path = Path(base_path)+ ensure_directory(base_path.parent)++ self.logger.debug(f"Exporting knowledge graph to Parquet: {base_path}")++ # Track KG export+ tracking_id = self.progress_tracker.start_tracking(+ file=str(base_path),+ module="export",+ submodule="ParquetExporter",+ message=f"Exporting knowledge graph to Parquet: {base_path}",+ )++ try:+ exported_files = []++ # Export entities+ if "entities" in kg and kg["entities"]:+ entities_path = base_path.parent / f"{base_path.stem}_entities.parquet"+ self.export_entities(kg["entities"], entities_path, **options)+ exported_files.append(entities_path)+ self.logger.info(f"Exported entities to: {entities_path}")++ # Export relationships+ if "relationships" in kg and kg["relationships"]:+ rels_path = base_path.parent / f"{base_path.stem}_relationships.parquet"+ self.export_relationships(kg["relationships"], rels_path, **options)+ exported_files.append(rels_path)+ self.logger.info(f"Exported relationships to: {rels_path}")++ self.logger.info(+ f"Exported knowledge graph to {len(exported_files)} Parquet file(s)"+ )+ self.progress_tracker.stop_tracking(+ tracking_id,+ status="completed",+ message=f"Exported {len(exported_files)} Parquet files",+ )++ except Exception as e:+ self.progress_tracker.stop_tracking(+ tracking_id, status="failed", message=str(e)+ )+ raise++ def _write_parquet(+ self,+ data: List[Dict[str, Any]],+ file_path: Union[str, Path],+ schema: Optional["pa.Schema"] = None,+ **options,+ ) -> None:+ """+ Write data to Parquet file.++ Internal method that handles the actual Parquet file writing using pyarrow.++ Args:+ data: List of dictionaries to write+ file_path: Output Parquet file path+ schema: Parquet schema to use (required for type safety)+ **options: Additional parquet write options++ Raises:+ ProcessingError: If Parquet writing fails+ ValidationError: If schema is not provided+ """+ if not schema:+ raise ValidationError("Schema is required for Parquet export")++ # Ensure file_path is a Path object+ file_path = Path(file_path)++ if not data:+ self.logger.warning(f"No data to write to {file_path}")+ # Write empty Parquet file with schema+ empty_table = pa.table({field.name: [] for field in schema}, schema=schema)+ pq.write_table(+ empty_table, str(file_path), compression=self.compression, **options+ )+ return++ try:+ # Create PyArrow table from data using explicit schema+ table = pa.Table.from_pylist(data, schema=schema)++ # Write to Parquet file+ pq.write_table(+ table, str(file_path), compression=self.compression, **options+ )++ file_size = file_path.stat().st_size+ self.logger.debug(+ f"Wrote {len(data)} row(s) to {file_path} ({file_size} bytes)"+ )++ except pa.ArrowInvalid as e:+ raise ProcessingError(+ f"Failed to create Parquet table: {e}. "+ "Check that data matches schema."+ )+ except Exception as e:+ raise ProcessingError(f"Failed to write Parquet file: {e}")diff --git a/tests/test_arango_aql_exporter.py b/tests/test_arango_aql_exporter.pyindex 36e2e000..9a5bb570 100644--- a/tests/test_arango_aql_exporter.py+++ b/tests/test_arango_aql_exporter.py@@ -1,538 +1,538 @@-"""-Tests for ArangoDB AQL Exporter Module--This module contains comprehensive tests for the ArangoDB AQL exporter,-validating AQL syntax generation, node and edge handling, and edge cases.-"""--import json-import re-import shutil-import tempfile-import unittest-from pathlib import Path--from semantica.export import ArangoAQLExporter---class TestArangoAQLExporter(unittest.TestCase):- """Test cases for ArangoDB AQL Exporter."""-- def setUp(self):- """Set up test fixtures."""- self.test_dir = tempfile.mkdtemp()-- # Sample entities for testing- self.entities = [- {- "id": "e1",- "type": "Person",- "name": "Alice",- "label": "Alice",- "properties": {"age": 30, "email": "[email protected]"},- },- {- "id": "e2",- "type": "Organization",- "name": "Acme Corp",- "label": "Acme Corp",- "properties": {"location": "New York", "founded": 2010},- },- {- "id": "e3",- "type": "Person",- "name": "Bob",- "label": "Bob",- "properties": {"age": 25},- },- ]-- # Sample relationships for testing- self.relationships = [- {- "id": "r1",- "source": "e1",- "target": "e2",- "type": "WORKS_FOR",- "properties": {"role": "Engineer", "since": 2020},- },- {- "id": "r2",- "source": "e3",- "target": "e2",- "type": "WORKS_FOR",- "properties": {"role": "Manager"},- },- {- "id": "r3",- "source": "e1",- "target": "e3",- "type": "KNOWS",- },- ]-- # Complete knowledge graph- self.kg = {- "entities": self.entities,- "relationships": self.relationships,- "metadata": {"version": "1.0", "created": "2024-01-01"},- }-- def tearDown(self):- """Clean up test fixtures."""- shutil.rmtree(self.test_dir)-- def test_exporter_initialization(self):- """Test exporter initialization with default and custom parameters."""- # Default initialization- exporter = ArangoAQLExporter()- self.assertEqual(exporter.vertex_collection, "vertices")- self.assertEqual(exporter.edge_collection, "edges")- self.assertEqual(exporter.batch_size, 1000)- self.assertTrue(exporter.include_collection_creation)-- # Custom initialization- exporter = ArangoAQLExporter(- vertex_collection="nodes",- edge_collection="links",- batch_size=500,- include_collection_creation=False,- )- self.assertEqual(exporter.vertex_collection, "nodes")- self.assertEqual(exporter.edge_collection, "links")- self.assertEqual(exporter.batch_size, 500)- self.assertFalse(exporter.include_collection_creation)-- def test_export_knowledge_graph(self):- """Test exporting a complete knowledge graph."""- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "graph.aql"-- exporter.export_knowledge_graph(self.kg, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Check for collection creation comments- self.assertIn("Create vertex collection", content)- self.assertIn("Create edge collection", content)-- # Check for INSERT statements- self.assertIn("INSERT doc INTO vertices", content)- self.assertIn("INSERT doc INTO edges", content)-- # Check for entity data- self.assertIn("Alice", content)- self.assertIn("Acme Corp", content)- self.assertIn("Bob", content)- self.assertIn("Person", content)- self.assertIn("Organization", content)-- # Check for relationship data- self.assertIn("WORKS_FOR", content)- self.assertIn("KNOWS", content)- self.assertIn("vertices/e1", content)- self.assertIn("vertices/e2", content)-- def test_export_entities_only(self):- """Test exporting only entities (vertices)."""- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "entities.aql"-- exporter.export_entities(self.entities, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Check for vertices INSERT- self.assertIn("INSERT doc INTO vertices", content)- self.assertIn("Alice", content)- self.assertIn("Acme Corp", content)-- # Check that edges are NOT present (empty relationships)- self.assertIn("Attempting to insert 0 edges", content)-- def test_export_relationships_only(self):- """Test exporting only relationships (edges)."""- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "relationships.aql"-- exporter.export_relationships(self.relationships, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Check for edges INSERT- self.assertIn("INSERT doc INTO edges", content)- self.assertIn("WORKS_FOR", content)- self.assertIn("KNOWS", content)-- # Check that vertices section indicates 0 vertices- self.assertIn("Inserting 0 vertices", content)-- def test_custom_collection_names(self):- """Test exporting with custom collection names."""- exporter = ArangoAQLExporter(- vertex_collection="custom_nodes", edge_collection="custom_edges"- )- output_path = Path(self.test_dir) / "custom.aql"-- exporter.export(self.kg, str(output_path))-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Check for custom collection names- self.assertIn("INSERT doc INTO custom_nodes", content)- self.assertIn("INSERT doc INTO custom_edges", content)- self.assertIn("custom_nodes/e1", content)-- def test_special_characters_in_properties(self):- """Test handling of special characters in node and edge properties."""- special_entities = [- {- "id": "special_1",- "type": "Person",- "name": "O'Brien",- "properties": {"quote": 'She said "hello"', "path": "C:\\Users\\test"},- }- ]-- special_relationships = [- {- "id": "special_r1",- "source": "special_1",- "target": "e1",- "type": "KNOWS",- "properties": {"note": "Uses 'quotes' and \"escapes\""},- }- ]-- kg = {"entities": special_entities, "relationships": special_relationships}-- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "special_chars.aql"-- exporter.export(kg, str(output_path))-- # Verify file was created and is valid JSON structure- self.assertTrue(output_path.exists())-- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # The content should contain the special characters properly escaped in JSON- self.assertIn("O'Brien", content)- self.assertIn("She said", content)-- def test_empty_collections(self):- """Test handling of empty entity and relationship collections."""- empty_kg = {"entities": [], "relationships": []}-- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "empty.aql"-- exporter.export(empty_kg, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Should have collection creation comments but no INSERT statements- self.assertIn("Inserting 0 vertices", content)- self.assertIn("Attempting to insert 0 edges", content)-- def test_missing_source_or_target(self):- """Test handling of edges with missing source or target."""- invalid_relationships = [- {"id": "r_invalid_1", "source": "e1", "type": "KNOWS"}, # Missing target- {"id": "r_invalid_2", "target": "e2", "type": "RELATED"}, # Missing source- {- "id": "r_valid",- "source": "e1",- "target": "e2",- "type": "VALID",- }, # Valid- ]-- kg = {"entities": self.entities, "relationships": invalid_relationships}-- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "invalid_edges.aql"-- # Should not raise an exception, but should skip invalid edges- exporter.export(kg, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # The data section should only contain the valid relationship- self.assertIn("VALID", content)- # The header comment should count all input edges, including invalid ones- self.assertIn("Attempting to insert 3 edges", content)-- def test_key_sanitization(self):- """Test sanitization of keys with invalid characters."""- entities_with_invalid_keys = [- {- "id": "[email protected]",- "type": "Email",- "name": "Test Email",- },- {- "id": "user/123/profile",- "type": "Profile",- "name": "User Profile",- },- {- "id": "_system",- "type": "System",- "name": "System Node",- },- ]-- kg = {"entities": entities_with_invalid_keys, "relationships": []}-- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "sanitized.aql"-- exporter.export(kg, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Keys should be sanitized (@ and / replaced with _)- self.assertIn("e1_domain_com", content)- self.assertIn("user_123_profile", content)- # _system should become k_system (no leading underscore)- self.assertIn("k_system", content)-- def test_batch_processing(self):- """Test batch processing with small batch size."""- # Create many entities to test batching- many_entities = [- {"id": f"e{i}", "type": "Node", "name": f"Node {i}"} for i in range(250)- ]-- kg = {"entities": many_entities, "relationships": []}-- # Use small batch size- exporter = ArangoAQLExporter(batch_size=100)- output_path = Path(self.test_dir) / "batched.aql"-- exporter.export(kg, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Count number of INSERT statements (should be 3 batches: 100, 100, 50)- insert_count = content.count("INSERT doc INTO vertices")- self.assertEqual(insert_count, 3, "Should have 3 batches for 250 entities")-- def test_nested_properties(self):- """Test handling of nested dictionaries and lists in properties."""- entities_with_nested = [- {- "id": "complex_1",- "type": "ComplexNode",- "name": "Complex",- "properties": {- "nested_dict": {"key1": "value1", "key2": "value2"},- "nested_list": [1, 2, 3, 4, 5],- "mixed": {"list": [1, 2], "value": "test"},- },- }- ]-- kg = {"entities": entities_with_nested, "relationships": []}-- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "nested.aql"-- exporter.export(kg, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Nested structures should be properly serialized as JSON- self.assertIn("nested_dict", content)- self.assertIn("nested_list", content)- # Check for the list values (may be formatted on separate lines)- self.assertIn("1", content)- self.assertIn("2", content)- self.assertIn("3", content)- self.assertIn("4", content)- self.assertIn("5", content)-- def test_aql_syntax_validity(self):- """Test that generated AQL has valid syntax structure."""- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "syntax_test.aql"-- exporter.export(self.kg, str(output_path))-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Check for valid AQL structure- # Should have FOR doc IN [array] INSERT doc INTO collection pattern- # Using re.search with DOTALL flag- pattern = r"FOR doc IN \[.*?\]\s+INSERT doc INTO \w+"- self.assertTrue(- re.search(pattern, content, re.DOTALL),- f"Pattern '{pattern}' not found in generated AQL",- )-- # Check that JSON arrays in the INSERT statements are valid- # Find all JSON arrays in the content- json_arrays = re.findall(r"FOR doc IN (\[.*?\])\s+INSERT", content, re.DOTALL)- for json_array_match in json_arrays:- # Extract just the array part- json_str = json_array_match.replace("\n INSERT", "").strip()- try:- # This should parse without errors- parsed = json.loads(json_str)- self.assertIsInstance(parsed, list)- except json.JSONDecodeError as e:- self.fail(f"Invalid JSON in AQL: {e}")-- def test_without_collection_creation(self):- """Test export without collection creation statements."""- exporter = ArangoAQLExporter(include_collection_creation=False)- output_path = Path(self.test_dir) / "no_creation.aql"-- exporter.export(self.kg, str(output_path))-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Should still have INSERT statements- self.assertIn("INSERT doc INTO vertices", content)- self.assertIn("INSERT doc INTO edges", content)-- def test_export_with_nodes_edges_keys(self):- """Test export using 'nodes' and 'edges' keys.-- Instead of 'entities' and 'relationships'.- """- kg_alt = {- "nodes": self.entities,- "edges": self.relationships,- }-- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "nodes_edges.aql"-- exporter.export(kg_alt, str(output_path))-- # Verify file was created- self.assertTrue(output_path.exists())-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Should have all the data- self.assertIn("Alice", content)- self.assertIn("WORKS_FOR", content)-- def test_override_collection_names_in_export(self):- """Test overriding collection names via export options."""- exporter = ArangoAQLExporter(- vertex_collection="default_v", edge_collection="default_e"- )- output_path = Path(self.test_dir) / "override.aql"-- # Override via options- exporter.export(- self.kg,- str(output_path),- vertex_collection="override_v",- edge_collection="override_e",- )-- # Read and verify content- with open(output_path, "r", encoding="utf-8") as f:- content = f.read()-- # Should use overridden names- self.assertIn("INSERT doc INTO override_v", content)- self.assertIn("INSERT doc INTO override_e", content)- self.assertIn("override_v/e1", content)-- def test_invalid_collection_name_on_init(self):- """Test that invalid collection names raise ValueError on initialization."""- # Test collection name starting with number- with self.assertRaises(ValueError) as context:- ArangoAQLExporter(vertex_collection="123invalid")- self.assertIn("must start with a letter or underscore", str(context.exception))-- # Test collection name with invalid characters- with self.assertRaises(ValueError) as context:- ArangoAQLExporter(edge_collection="invalid@name")- self.assertIn("contains invalid character", str(context.exception))-- # Test empty collection name- with self.assertRaises(ValueError) as context:- ArangoAQLExporter(vertex_collection="")- self.assertIn("cannot be empty", str(context.exception))-- # Test too long collection name- with self.assertRaises(ValueError) as context:- ArangoAQLExporter(vertex_collection="a" * 257)- self.assertIn("exceeds maximum length", str(context.exception))-- def test_invalid_collection_name_on_export(self):- """Test invalid collection names raise ValueError when overriding."""- exporter = ArangoAQLExporter()- output_path = Path(self.test_dir) / "test.aql"-- kg = {"entities": self.entities, "relationships": self.relationships}-- # Test invalid vertex collection override- with self.assertRaises(ValueError) as context:- exporter.export(kg, str(output_path), vertex_collection="123invalid")- self.assertIn("must start with a letter or underscore", str(context.exception))-- # Test invalid edge collection override- with self.assertRaises(ValueError) as context:- exporter.export(kg, str(output_path), edge_collection="invalid@name")- self.assertIn("contains invalid character", str(context.exception))---if __name__ == "__main__":- unittest.main()+"""+Tests for ArangoDB AQL Exporter Module++This module contains comprehensive tests for the ArangoDB AQL exporter,+validating AQL syntax generation, node and edge handling, and edge cases.+"""++import json+import re+import shutil+import tempfile+import unittest+from pathlib import Path++from semantica.export import ArangoAQLExporter+++class TestArangoAQLExporter(unittest.TestCase):+ """Test cases for ArangoDB AQL Exporter."""++ def setUp(self):+ """Set up test fixtures."""+ self.test_dir = tempfile.mkdtemp()++ # Sample entities for testing+ self.entities = [+ {+ "id": "e1",+ "type": "Person",+ "name": "Alice",+ "label": "Alice",+ "properties": {"age": 30, "email": "[email protected]"},+ },+ {+ "id": "e2",+ "type": "Organization",+ "name": "Acme Corp",+ "label": "Acme Corp",+ "properties": {"location": "New York", "founded": 2010},+ },+ {+ "id": "e3",+ "type": "Person",+ "name": "Bob",+ "label": "Bob",+ "properties": {"age": 25},+ },+ ]++ # Sample relationships for testing+ self.relationships = [+ {+ "id": "r1",+ "source": "e1",+ "target": "e2",+ "type": "WORKS_FOR",+ "properties": {"role": "Engineer", "since": 2020},+ },+ {+ "id": "r2",+ "source": "e3",+ "target": "e2",+ "type": "WORKS_FOR",+ "properties": {"role": "Manager"},+ },+ {+ "id": "r3",+ "source": "e1",+ "target": "e3",+ "type": "KNOWS",+ },+ ]++ # Complete knowledge graph+ self.kg = {+ "entities": self.entities,+ "relationships": self.relationships,+ "metadata": {"version": "1.0", "created": "2024-01-01"},+ }++ def tearDown(self):+ """Clean up test fixtures."""+ shutil.rmtree(self.test_dir)++ def test_exporter_initialization(self):+ """Test exporter initialization with default and custom parameters."""+ # Default initialization+ exporter = ArangoAQLExporter()+ self.assertEqual(exporter.vertex_collection, "vertices")+ self.assertEqual(exporter.edge_collection, "edges")+ self.assertEqual(exporter.batch_size, 1000)+ self.assertTrue(exporter.include_collection_creation)++ # Custom initialization+ exporter = ArangoAQLExporter(+ vertex_collection="nodes",+ edge_collection="links",+ batch_size=500,+ include_collection_creation=False,+ )+ self.assertEqual(exporter.vertex_collection, "nodes")+ self.assertEqual(exporter.edge_collection, "links")+ self.assertEqual(exporter.batch_size, 500)+ self.assertFalse(exporter.include_collection_creation)++ def test_export_knowledge_graph(self):+ """Test exporting a complete knowledge graph."""+ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "graph.aql"++ exporter.export_knowledge_graph(self.kg, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Check for collection creation comments+ self.assertIn("Create vertex collection", content)+ self.assertIn("Create edge collection", content)++ # Check for INSERT statements+ self.assertIn("INSERT doc INTO vertices", content)+ self.assertIn("INSERT doc INTO edges", content)++ # Check for entity data+ self.assertIn("Alice", content)+ self.assertIn("Acme Corp", content)+ self.assertIn("Bob", content)+ self.assertIn("Person", content)+ self.assertIn("Organization", content)++ # Check for relationship data+ self.assertIn("WORKS_FOR", content)+ self.assertIn("KNOWS", content)+ self.assertIn("vertices/e1", content)+ self.assertIn("vertices/e2", content)++ def test_export_entities_only(self):+ """Test exporting only entities (vertices)."""+ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "entities.aql"++ exporter.export_entities(self.entities, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Check for vertices INSERT+ self.assertIn("INSERT doc INTO vertices", content)+ self.assertIn("Alice", content)+ self.assertIn("Acme Corp", content)++ # Check that edges are NOT present (empty relationships)+ self.assertIn("Attempting to insert 0 edges", content)++ def test_export_relationships_only(self):+ """Test exporting only relationships (edges)."""+ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "relationships.aql"++ exporter.export_relationships(self.relationships, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Check for edges INSERT+ self.assertIn("INSERT doc INTO edges", content)+ self.assertIn("WORKS_FOR", content)+ self.assertIn("KNOWS", content)++ # Check that vertices section indicates 0 vertices+ self.assertIn("Inserting 0 vertices", content)++ def test_custom_collection_names(self):+ """Test exporting with custom collection names."""+ exporter = ArangoAQLExporter(+ vertex_collection="custom_nodes", edge_collection="custom_edges"+ )+ output_path = Path(self.test_dir) / "custom.aql"++ exporter.export(self.kg, str(output_path))++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Check for custom collection names+ self.assertIn("INSERT doc INTO custom_nodes", content)+ self.assertIn("INSERT doc INTO custom_edges", content)+ self.assertIn("custom_nodes/e1", content)++ def test_special_characters_in_properties(self):+ """Test handling of special characters in node and edge properties."""+ special_entities = [+ {+ "id": "special_1",+ "type": "Person",+ "name": "O'Brien",+ "properties": {"quote": 'She said "hello"', "path": "C:\\Users\\test"},+ }+ ]++ special_relationships = [+ {+ "id": "special_r1",+ "source": "special_1",+ "target": "e1",+ "type": "KNOWS",+ "properties": {"note": "Uses 'quotes' and \"escapes\""},+ }+ ]++ kg = {"entities": special_entities, "relationships": special_relationships}++ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "special_chars.aql"++ exporter.export(kg, str(output_path))++ # Verify file was created and is valid JSON structure+ self.assertTrue(output_path.exists())++ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # The content should contain the special characters properly escaped in JSON+ self.assertIn("O'Brien", content)+ self.assertIn("She said", content)++ def test_empty_collections(self):+ """Test handling of empty entity and relationship collections."""+ empty_kg = {"entities": [], "relationships": []}++ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "empty.aql"++ exporter.export(empty_kg, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Should have collection creation comments but no INSERT statements+ self.assertIn("Inserting 0 vertices", content)+ self.assertIn("Attempting to insert 0 edges", content)++ def test_missing_source_or_target(self):+ """Test handling of edges with missing source or target."""+ invalid_relationships = [+ {"id": "r_invalid_1", "source": "e1", "type": "KNOWS"}, # Missing target+ {"id": "r_invalid_2", "target": "e2", "type": "RELATED"}, # Missing source+ {+ "id": "r_valid",+ "source": "e1",+ "target": "e2",+ "type": "VALID",+ }, # Valid+ ]++ kg = {"entities": self.entities, "relationships": invalid_relationships}++ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "invalid_edges.aql"++ # Should not raise an exception, but should skip invalid edges+ exporter.export(kg, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # The data section should only contain the valid relationship+ self.assertIn("VALID", content)+ # The header comment should count all input edges, including invalid ones+ self.assertIn("Attempting to insert 3 edges", content)++ def test_key_sanitization(self):+ """Test sanitization of keys with invalid characters."""+ entities_with_invalid_keys = [+ {+ "id": "[email protected]",+ "type": "Email",+ "name": "Test Email",+ },+ {+ "id": "user/123/profile",+ "type": "Profile",+ "name": "User Profile",+ },+ {+ "id": "_system",+ "type": "System",+ "name": "System Node",+ },+ ]++ kg = {"entities": entities_with_invalid_keys, "relationships": []}++ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "sanitized.aql"++ exporter.export(kg, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Keys should be sanitized (@ and / replaced with _)+ self.assertIn("e1_domain_com", content)+ self.assertIn("user_123_profile", content)+ # _system should become k_system (no leading underscore)+ self.assertIn("k_system", content)++ def test_batch_processing(self):+ """Test batch processing with small batch size."""+ # Create many entities to test batching+ many_entities = [+ {"id": f"e{i}", "type": "Node", "name": f"Node {i}"} for i in range(250)+ ]++ kg = {"entities": many_entities, "relationships": []}++ # Use small batch size+ exporter = ArangoAQLExporter(batch_size=100)+ output_path = Path(self.test_dir) / "batched.aql"++ exporter.export(kg, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Count number of INSERT statements (should be 3 batches: 100, 100, 50)+ insert_count = content.count("INSERT doc INTO vertices")+ self.assertEqual(insert_count, 3, "Should have 3 batches for 250 entities")++ def test_nested_properties(self):+ """Test handling of nested dictionaries and lists in properties."""+ entities_with_nested = [+ {+ "id": "complex_1",+ "type": "ComplexNode",+ "name": "Complex",+ "properties": {+ "nested_dict": {"key1": "value1", "key2": "value2"},+ "nested_list": [1, 2, 3, 4, 5],+ "mixed": {"list": [1, 2], "value": "test"},+ },+ }+ ]++ kg = {"entities": entities_with_nested, "relationships": []}++ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "nested.aql"++ exporter.export(kg, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Nested structures should be properly serialized as JSON+ self.assertIn("nested_dict", content)+ self.assertIn("nested_list", content)+ # Check for the list values (may be formatted on separate lines)+ self.assertIn("1", content)+ self.assertIn("2", content)+ self.assertIn("3", content)+ self.assertIn("4", content)+ self.assertIn("5", content)++ def test_aql_syntax_validity(self):+ """Test that generated AQL has valid syntax structure."""+ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "syntax_test.aql"++ exporter.export(self.kg, str(output_path))++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Check for valid AQL structure+ # Should have FOR doc IN [array] INSERT doc INTO collection pattern+ # Using re.search with DOTALL flag+ pattern = r"FOR doc IN \[.*?\]\s+INSERT doc INTO \w+"+ self.assertTrue(+ re.search(pattern, content, re.DOTALL),+ f"Pattern '{pattern}' not found in generated AQL",+ )++ # Check that JSON arrays in the INSERT statements are valid+ # Find all JSON arrays in the content+ json_arrays = re.findall(r"FOR doc IN (\[.*?\])\s+INSERT", content, re.DOTALL)+ for json_array_match in json_arrays:+ # Extract just the array part+ json_str = json_array_match.replace("\n INSERT", "").strip()+ try:+ # This should parse without errors+ parsed = json.loads(json_str)+ self.assertIsInstance(parsed, list)+ except json.JSONDecodeError as e:+ self.fail(f"Invalid JSON in AQL: {e}")++ def test_without_collection_creation(self):+ """Test export without collection creation statements."""+ exporter = ArangoAQLExporter(include_collection_creation=False)+ output_path = Path(self.test_dir) / "no_creation.aql"++ exporter.export(self.kg, str(output_path))++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Should still have INSERT statements+ self.assertIn("INSERT doc INTO vertices", content)+ self.assertIn("INSERT doc INTO edges", content)++ def test_export_with_nodes_edges_keys(self):+ """Test export using 'nodes' and 'edges' keys.++ Instead of 'entities' and 'relationships'.+ """+ kg_alt = {+ "nodes": self.entities,+ "edges": self.relationships,+ }++ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "nodes_edges.aql"++ exporter.export(kg_alt, str(output_path))++ # Verify file was created+ self.assertTrue(output_path.exists())++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Should have all the data+ self.assertIn("Alice", content)+ self.assertIn("WORKS_FOR", content)++ def test_override_collection_names_in_export(self):+ """Test overriding collection names via export options."""+ exporter = ArangoAQLExporter(+ vertex_collection="default_v", edge_collection="default_e"+ )+ output_path = Path(self.test_dir) / "override.aql"++ # Override via options+ exporter.export(+ self.kg,+ str(output_path),+ vertex_collection="override_v",+ edge_collection="override_e",+ )++ # Read and verify content+ with open(output_path, "r", encoding="utf-8") as f:+ content = f.read()++ # Should use overridden names+ self.assertIn("INSERT doc INTO override_v", content)+ self.assertIn("INSERT doc INTO override_e", content)+ self.assertIn("override_v/e1", content)++ def test_invalid_collection_name_on_init(self):+ """Test that invalid collection names raise ValueError on initialization."""+ # Test collection name starting with number+ with self.assertRaises(ValueError) as context:+ ArangoAQLExporter(vertex_collection="123invalid")+ self.assertIn("must start with a letter or underscore", str(context.exception))++ # Test collection name with invalid characters+ with self.assertRaises(ValueError) as context:+ ArangoAQLExporter(edge_collection="invalid@name")+ self.assertIn("contains invalid character", str(context.exception))++ # Test empty collection name+ with self.assertRaises(ValueError) as context:+ ArangoAQLExporter(vertex_collection="")+ self.assertIn("cannot be empty", str(context.exception))++ # Test too long collection name+ with self.assertRaises(ValueError) as context:+ ArangoAQLExporter(vertex_collection="a" * 257)+ self.assertIn("exceeds maximum length", str(context.exception))++ def test_invalid_collection_name_on_export(self):+ """Test invalid collection names raise ValueError when overriding."""+ exporter = ArangoAQLExporter()+ output_path = Path(self.test_dir) / "test.aql"++ kg = {"entities": self.entities, "relationships": self.relationships}++ # Test invalid vertex collection override+ with self.assertRaises(ValueError) as context:+ exporter.export(kg, str(output_path), vertex_collection="123invalid")+ self.assertIn("must start with a letter or underscore", str(context.exception))++ # Test invalid edge collection override+ with self.assertRaises(ValueError) as context:+ exporter.export(kg, str(output_path), edge_collection="invalid@name")+ self.assertIn("contains invalid character", str(context.exception))+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_parquet_exporter.py b/tests/test_parquet_exporter.pyindex 83ca3a35..f8782afa 100644--- a/tests/test_parquet_exporter.py+++ b/tests/test_parquet_exporter.py@@ -1,546 +1,546 @@-"""-Unit tests for Apache Parquet exporter module.--Tests schema validation, data export, pandas conversion, empty inputs,-and minimal graph structures.-"""--import shutil-import tempfile-import unittest-from pathlib import Path--# Try to import pyarrow-try:- import pyarrow.parquet as pq-- PARQUET_AVAILABLE = True-except ImportError:- PARQUET_AVAILABLE = False--from semantica.export import ParquetExporter-from semantica.utils.exceptions import ValidationError--[email protected](not PARQUET_AVAILABLE, "pyarrow not installed")-class TestParquetExporter(unittest.TestCase):- """Test cases for ParquetExporter class."""-- def setUp(self):- """Set up test fixtures."""- self.test_dir = tempfile.mkdtemp()-- # Sample entities with various field names- self.entities = [- {- "id": "e1",- "type": "Person",- "name": "Alice",- "label": "Alice",- "confidence": 0.95,- "start": 0,- "end": 5,- "metadata": {"age": 30, "city": "NYC"},- },- {- "id": "e2",- "type": "Organization",- "text": "Acme Corp",- "entity_type": "ORG",- "confidence": 0.88,- "start_offset": 10,- "end_offset": 19,- "metadata": {"location": "NY", "employees": 100},- },- ]-- # Sample relationships- self.relationships = [- {- "id": "r1",- "source": "e1",- "target": "e2",- "type": "WORKS_FOR",- "confidence": 0.92,- "metadata": {"role": "Engineer", "since": 2020},- },- {- "source_id": "e2",- "target_id": "e1",- "relationship_type": "EMPLOYS",- "confidence": 0.90,- },- ]-- # Knowledge graph- self.kg = {- "entities": self.entities,- "relationships": self.relationships,- "metadata": {"version": "1.0", "created": "2024-01-01"},- }-- def tearDown(self):- """Clean up test directory."""- shutil.rmtree(self.test_dir)-- def test_initialization(self):- """Test ParquetExporter initialization."""- exporter = ParquetExporter()- self.assertIsNotNone(exporter)- self.assertEqual(exporter.compression, "snappy")-- # Test with different compression- exporter_gzip = ParquetExporter(compression="gzip")- self.assertEqual(exporter_gzip.compression, "gzip")-- exporter_none = ParquetExporter(compression="none")- self.assertIsNone(exporter_none.compression)-- def test_initialization_without_pyarrow(self):- """Test initialization fails gracefully without pyarrow."""- # This test verifies the constant is correctly set- if not PARQUET_AVAILABLE:- self.assertFalse(PARQUET_AVAILABLE)-- def test_export_entities_basic(self):- """Test basic entity export to Parquet."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "entities.parquet"-- exporter.export_entities(self.entities, str(output_path))- self.assertTrue(output_path.exists())-- # Read and verify Parquet file- table = pq.read_table(str(output_path))-- # Verify schema- from semantica.export.parquet_exporter import ENTITY_SCHEMA-- self.assertEqual(table.schema, ENTITY_SCHEMA)-- # Verify data- self.assertEqual(table.num_rows, 2)- self.assertEqual(table.column("id")[0].as_py(), "e1")- self.assertEqual(table.column("type")[0].as_py(), "Person")- self.assertEqual(table.column("confidence")[0].as_py(), 0.95)-- def test_export_entities_field_normalization(self):- """Test entity field name normalization."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "entities_normalized.parquet"-- # Entities with various field name variations- varied_entities = [- {"id": "e1", "text": "Entity 1", "type": "TYPE1"},- {"entity_id": "e2", "label": "Entity 2", "entity_type": "TYPE2"},- {"id": "e3", "name": "Entity 3", "type": "TYPE3"},- ]-- exporter.export_entities(varied_entities, str(output_path))- self.assertTrue(output_path.exists())-- # Read and verify normalization- table = pq.read_table(str(output_path))-- self.assertEqual(table.num_rows, 3)- self.assertEqual(table.column("id")[0].as_py(), "e1")- self.assertEqual(table.column("text")[0].as_py(), "Entity 1")- self.assertEqual(table.column("text")[1].as_py(), "Entity 2")- self.assertEqual(table.column("text")[2].as_py(), "Entity 3")-- def test_export_entities_with_compression(self):- """Test entity export with different compression codecs."""- import pyarrow.parquet as pq_module-- for compression in ["snappy", "gzip", "brotli", "zstd", "lz4", "none"]:- with self.subTest(compression=compression):- # Check if codec is available in this pyarrow build- try:- # Test codec availability by checking compression opts- if compression != "none":- codec_available = compression.upper() in dir(- pq_module.lib.Codec- )- if not codec_available:- self.skipTest(- f"Codec {compression} not available in " "pyarrow build"- )- except AttributeError:- # If we can't check, just try and skip on error- pass-- try:- exporter = ParquetExporter(compression=compression)- output_path = (- Path(self.test_dir) / f"entities_{compression}.parquet"- )-- exporter.export_entities(self.entities, str(output_path))- self.assertTrue(output_path.exists())-- # Verify file can be read- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 2)- except (ImportError, RuntimeError, OSError) as e:- if "codec" in str(e).lower() or "compression" in str(e).lower():- self.skipTest(f"Codec {compression} not available: {e}")- raise-- def test_export_entities_empty(self):- """Test exporting empty entities list raises ValidationError."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "empty_entities.parquet"-- with self.assertRaises(ValidationError):- exporter.export_entities([], str(output_path))-- def test_export_entities_metadata_handling(self):- """Test entity metadata is correctly serialized."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "entities_metadata.parquet"-- exporter.export_entities(self.entities, str(output_path))-- # Read and verify metadata- table = pq.read_table(str(output_path))- metadata_col = table.column("metadata")-- # First entity should have metadata- first_metadata = metadata_col[0].as_py()- self.assertIsNotNone(first_metadata)- self.assertIn("keys", first_metadata)- self.assertIn("values", first_metadata)- self.assertEqual(set(first_metadata["keys"]), {"age", "city"})-- def test_export_relationships_basic(self):- """Test basic relationship export to Parquet."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "relationships.parquet"-- exporter.export_relationships(self.relationships, str(output_path))- self.assertTrue(output_path.exists())-- # Read and verify Parquet file- table = pq.read_table(str(output_path))-- # Verify schema- from semantica.export.parquet_exporter import RELATIONSHIP_SCHEMA-- self.assertEqual(table.schema, RELATIONSHIP_SCHEMA)-- # Verify data- self.assertEqual(table.num_rows, 2)- self.assertEqual(table.column("id")[0].as_py(), "r1")- self.assertEqual(table.column("source_id")[0].as_py(), "e1")- self.assertEqual(table.column("target_id")[0].as_py(), "e2")- self.assertEqual(table.column("type")[0].as_py(), "WORKS_FOR")-- def test_export_relationships_field_normalization(self):- """Test relationship field name normalization."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "relationships_normalized.parquet"-- # Relationships with various field name variations- varied_rels = [- {"id": "r1", "source": "e1", "target": "e2", "type": "TYPE1"},- {"source_id": "e2", "target_id": "e3", "relationship_type": "TYPE2"},- {"from_id": "e3", "to_id": "e1", "relation_type": "TYPE3"},- ]-- exporter.export_relationships(varied_rels, str(output_path))- self.assertTrue(output_path.exists())-- # Read and verify normalization- table = pq.read_table(str(output_path))-- self.assertEqual(table.num_rows, 3)- self.assertEqual(table.column("source_id")[0].as_py(), "e1")- self.assertEqual(table.column("target_id")[0].as_py(), "e2")-- def test_export_relationships_empty(self):- """Test exporting empty relationships list raises ValidationError."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "empty_relationships.parquet"-- with self.assertRaises(ValidationError):- exporter.export_relationships([], str(output_path))-- def test_export_relationships_auto_id_generation(self):- """Test relationship ID is auto-generated when missing."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "relationships_auto_id.parquet"-- # Relationships without IDs- rels_no_id = [- {"source_id": "e1", "target_id": "e2", "type": "REL1"},- {"source_id": "e2", "target_id": "e3", "type": "REL2"},- ]-- exporter.export_relationships(rels_no_id, str(output_path))-- # Read and verify IDs were generated- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 2)- self.assertIsNotNone(table.column("id")[0].as_py())- self.assertIsNotNone(table.column("id")[1].as_py())-- def test_export_knowledge_graph_basic(self):- """Test basic knowledge graph export to multiple Parquet files."""- exporter = ParquetExporter()- base_path = Path(self.test_dir) / "kg"-- exporter.export_knowledge_graph(self.kg, str(base_path))-- # Verify files were created- entities_path = Path(self.test_dir) / "kg_entities.parquet"- rels_path = Path(self.test_dir) / "kg_relationships.parquet"-- self.assertTrue(entities_path.exists())- self.assertTrue(rels_path.exists())-- # Verify entities- entities_table = pq.read_table(str(entities_path))- self.assertEqual(entities_table.num_rows, 2)-- # Verify relationships- rels_table = pq.read_table(str(rels_path))- self.assertEqual(rels_table.num_rows, 2)-- def test_export_knowledge_graph_invalid_input(self):- """Test knowledge graph export with invalid input raises ValidationError."""- exporter = ParquetExporter()- base_path = Path(self.test_dir) / "kg_invalid"-- # Not a dictionary- with self.assertRaises(ValidationError):- exporter.export_knowledge_graph("not a dict", str(base_path))-- # Missing both entities and relationships- with self.assertRaises(ValidationError):- exporter.export_knowledge_graph({"metadata": {}}, str(base_path))-- def test_export_knowledge_graph_partial(self):- """Test knowledge graph export with only entities or relationships."""- exporter = ParquetExporter()-- # Only entities- kg_entities_only = {"entities": self.entities}- base_path_ent = Path(self.test_dir) / "kg_entities_only"- exporter.export_knowledge_graph(kg_entities_only, str(base_path_ent))-- entities_path = Path(self.test_dir) / "kg_entities_only_entities.parquet"- self.assertTrue(entities_path.exists())-- # Only relationships- kg_rels_only = {"relationships": self.relationships}- base_path_rel = Path(self.test_dir) / "kg_rels_only"- exporter.export_knowledge_graph(kg_rels_only, str(base_path_rel))-- rels_path = Path(self.test_dir) / "kg_rels_only_relationships.parquet"- self.assertTrue(rels_path.exists())-- def test_export_generic_list(self):- """Test generic export with list of dictionaries."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "generic_list.parquet"-- exporter.export(self.entities, str(output_path), schema=None)- self.assertTrue(output_path.exists())-- # Verify file can be read (schema auto-selected based on data structure)- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 2)-- def test_export_generic_dict(self):- """Test generic export with dictionary (multiple files)."""- exporter = ParquetExporter()- base_path = Path(self.test_dir) / "generic_dict"-- data_dict = {"entities": self.entities, "relationships": self.relationships}-- exporter.export(data_dict, str(base_path))-- # Verify both files were created- entities_path = Path(self.test_dir) / "generic_dict_entities.parquet"- rels_path = Path(self.test_dir) / "generic_dict_relationships.parquet"-- self.assertTrue(entities_path.exists())- self.assertTrue(rels_path.exists())-- def test_export_invalid_data_type(self):- """Test export with invalid data type raises ValidationError."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "invalid.parquet"-- with self.assertRaises(ValidationError):- exporter.export("invalid string data", str(output_path))-- def test_pandas_compatibility(self):- """Test exported Parquet files can be read by pandas."""- try:- import pandas as pd- except ImportError:- self.skipTest("pandas not installed")-- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "pandas_test.parquet"-- exporter.export_entities(self.entities, str(output_path))-- # Read with pandas- df = pd.read_parquet(str(output_path))-- self.assertEqual(len(df), 2)- self.assertIn("id", df.columns)- self.assertIn("text", df.columns)- self.assertIn("type", df.columns)-- def test_file_size_comparison(self):- """Test Parquet file sizes with different compression."""- exporter_snappy = ParquetExporter(compression="snappy")- exporter_gzip = ParquetExporter(compression="gzip")- exporter_none = ParquetExporter(compression="none")-- path_snappy = Path(self.test_dir) / "size_snappy.parquet"- path_gzip = Path(self.test_dir) / "size_gzip.parquet"- path_none = Path(self.test_dir) / "size_none.parquet"-- # Create larger dataset for meaningful comparison- large_entities = self.entities * 100-- exporter_snappy.export_entities(large_entities, str(path_snappy))- exporter_gzip.export_entities(large_entities, str(path_gzip))- exporter_none.export_entities(large_entities, str(path_none))-- size_snappy = path_snappy.stat().st_size- size_none = path_none.stat().st_size-- # Uncompressed should be largest- self.assertGreater(size_none, size_snappy)-- # All files should be readable- for path in [path_snappy, path_gzip, path_none]:- table = pq.read_table(str(path))- self.assertEqual(table.num_rows, 200)-- def test_entity_missing_id_skipped(self):- """Test entities without IDs are skipped with warning."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "entities_missing_id.parquet"-- # Mix of entities with and without IDs- entities_mixed = [- {"id": "e1", "text": "Valid Entity"},- {"text": "Missing ID"}, # No ID- {"id": "e2", "text": "Another Valid"},- ]-- exporter.export_entities(entities_mixed, str(output_path))-- # Only 2 entities should be exported (one without ID is skipped)- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 2)-- def test_relationship_missing_source_target_skipped(self):- """Test relationships without source/target are skipped with warning."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "rels_missing.parquet"-- # Mix of valid and invalid relationships- rels_mixed = [- {"id": "r1", "source_id": "e1", "target_id": "e2"},- {"id": "r2", "target_id": "e2"}, # Missing source- {"id": "r3", "source_id": "e1"}, # Missing target- {"id": "r4", "source_id": "e3", "target_id": "e4"},- ]-- exporter.export_relationships(rels_mixed, str(output_path))-- # Only 2 valid relationships should be exported- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 2)-- def test_all_entities_skipped_raises_error(self):- """Test that exporting entities with all skipped raises ValidationError."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "all_skipped.parquet"-- # All entities missing IDs- bad_entities = [- {"text": "No ID 1"},- {"text": "No ID 2"},- "not a dict",- ]-- with self.assertRaises(ValidationError) as cm:- exporter.export_entities(bad_entities, str(output_path))-- self.assertIn("No valid entities", str(cm.exception))-- def test_all_relationships_skipped_raises_error(self):- """Test that exporting relationships with all skipped raises ValidationError."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "all_rels_skipped.parquet"-- # All relationships missing source or target- bad_rels = [- {"id": "r1", "source_id": "e1"}, # Missing target- {"id": "r2", "target_id": "e2"}, # Missing source- "not a dict",- ]-- with self.assertRaises(ValidationError) as cm:- exporter.export_relationships(bad_rels, str(output_path))-- self.assertIn("No valid relationships", str(cm.exception))-- def test_invalid_confidence_values_handled(self):- """Test that invalid confidence values are handled gracefully."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "invalid_confidence.parquet"-- entities_with_invalid_conf = [- {"id": "e1", "text": "Valid", "confidence": 0.9},- {"id": "e2", "text": "String conf", "confidence": "invalid"},- {"id": "e3", "text": "None conf", "confidence": None},- ]-- exporter.export_entities(entities_with_invalid_conf, str(output_path))-- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 3)- # First entity has valid confidence- self.assertEqual(table.column("confidence")[0].as_py(), 0.9)- # Second entity has invalid confidence (should be None)- self.assertIsNone(table.column("confidence")[1].as_py())- # Third entity has None confidence- self.assertIsNone(table.column("confidence")[2].as_py())-- def test_invalid_start_end_values_handled(self):- """Test that invalid start/end offset values are handled gracefully."""- exporter = ParquetExporter()- output_path = Path(self.test_dir) / "invalid_offsets.parquet"-- entities_with_invalid_offsets = [- {"id": "e1", "text": "Valid", "start": 0, "end": 10},- {"id": "e2", "text": "String offsets", "start": "abc", "end": "def"},- {"id": "e3", "text": "None offsets", "start": None, "end": None},- ]-- exporter.export_entities(entities_with_invalid_offsets, str(output_path))-- table = pq.read_table(str(output_path))- self.assertEqual(table.num_rows, 3)- # First entity has valid offsets- self.assertEqual(table.column("start")[0].as_py(), 0)- self.assertEqual(table.column("end")[0].as_py(), 10)- # Second entity has invalid offsets (should be None)- self.assertIsNone(table.column("start")[1].as_py())- self.assertIsNone(table.column("end")[1].as_py())- # Third entity has None offsets- self.assertIsNone(table.column("start")[2].as_py())- self.assertIsNone(table.column("end")[2].as_py())---if __name__ == "__main__":- unittest.main()+"""+Unit tests for Apache Parquet exporter module.++Tests schema validation, data export, pandas conversion, empty inputs,+and minimal graph structures.+"""++import shutil+import tempfile+import unittest+from pathlib import Path++# Try to import pyarrow+try:+ import pyarrow.parquet as pq++ PARQUET_AVAILABLE = True+except ImportError:+ PARQUET_AVAILABLE = False++from semantica.export import ParquetExporter+from semantica.utils.exceptions import ValidationError++[email protected](not PARQUET_AVAILABLE, "pyarrow not installed")+class TestParquetExporter(unittest.TestCase):+ """Test cases for ParquetExporter class."""++ def setUp(self):+ """Set up test fixtures."""+ self.test_dir = tempfile.mkdtemp()++ # Sample entities with various field names+ self.entities = [+ {+ "id": "e1",+ "type": "Person",+ "name": "Alice",+ "label": "Alice",+ "confidence": 0.95,+ "start": 0,+ "end": 5,+ "metadata": {"age": 30, "city": "NYC"},+ },+ {+ "id": "e2",+ "type": "Organization",+ "text": "Acme Corp",+ "entity_type": "ORG",+ "confidence": 0.88,+ "start_offset": 10,+ "end_offset": 19,+ "metadata": {"location": "NY", "employees": 100},+ },+ ]++ # Sample relationships+ self.relationships = [+ {+ "id": "r1",+ "source": "e1",+ "target": "e2",+ "type": "WORKS_FOR",+ "confidence": 0.92,+ "metadata": {"role": "Engineer", "since": 2020},+ },+ {+ "source_id": "e2",+ "target_id": "e1",+ "relationship_type": "EMPLOYS",+ "confidence": 0.90,+ },+ ]++ # Knowledge graph+ self.kg = {+ "entities": self.entities,+ "relationships": self.relationships,+ "metadata": {"version": "1.0", "created": "2024-01-01"},+ }++ def tearDown(self):+ """Clean up test directory."""+ shutil.rmtree(self.test_dir)++ def test_initialization(self):+ """Test ParquetExporter initialization."""+ exporter = ParquetExporter()+ self.assertIsNotNone(exporter)+ self.assertEqual(exporter.compression, "snappy")++ # Test with different compression+ exporter_gzip = ParquetExporter(compression="gzip")+ self.assertEqual(exporter_gzip.compression, "gzip")++ exporter_none = ParquetExporter(compression="none")+ self.assertIsNone(exporter_none.compression)++ def test_initialization_without_pyarrow(self):+ """Test initialization fails gracefully without pyarrow."""+ # This test verifies the constant is correctly set+ if not PARQUET_AVAILABLE:+ self.assertFalse(PARQUET_AVAILABLE)++ def test_export_entities_basic(self):+ """Test basic entity export to Parquet."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "entities.parquet"++ exporter.export_entities(self.entities, str(output_path))+ self.assertTrue(output_path.exists())++ # Read and verify Parquet file+ table = pq.read_table(str(output_path))++ # Verify schema+ from semantica.export.parquet_exporter import ENTITY_SCHEMA++ self.assertEqual(table.schema, ENTITY_SCHEMA)++ # Verify data+ self.assertEqual(table.num_rows, 2)+ self.assertEqual(table.column("id")[0].as_py(), "e1")+ self.assertEqual(table.column("type")[0].as_py(), "Person")+ self.assertEqual(table.column("confidence")[0].as_py(), 0.95)++ def test_export_entities_field_normalization(self):+ """Test entity field name normalization."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "entities_normalized.parquet"++ # Entities with various field name variations+ varied_entities = [+ {"id": "e1", "text": "Entity 1", "type": "TYPE1"},+ {"entity_id": "e2", "label": "Entity 2", "entity_type": "TYPE2"},+ {"id": "e3", "name": "Entity 3", "type": "TYPE3"},+ ]++ exporter.export_entities(varied_entities, str(output_path))+ self.assertTrue(output_path.exists())++ # Read and verify normalization+ table = pq.read_table(str(output_path))++ self.assertEqual(table.num_rows, 3)+ self.assertEqual(table.column("id")[0].as_py(), "e1")+ self.assertEqual(table.column("text")[0].as_py(), "Entity 1")+ self.assertEqual(table.column("text")[1].as_py(), "Entity 2")+ self.assertEqual(table.column("text")[2].as_py(), "Entity 3")++ def test_export_entities_with_compression(self):+ """Test entity export with different compression codecs."""+ import pyarrow.parquet as pq_module++ for compression in ["snappy", "gzip", "brotli", "zstd", "lz4", "none"]:+ with self.subTest(compression=compression):+ # Check if codec is available in this pyarrow build+ try:+ # Test codec availability by checking compression opts+ if compression != "none":+ codec_available = compression.upper() in dir(+ pq_module.lib.Codec+ )+ if not codec_available:+ self.skipTest(+ f"Codec {compression} not available in " "pyarrow build"+ )+ except AttributeError:+ # If we can't check, just try and skip on error+ pass++ try:+ exporter = ParquetExporter(compression=compression)+ output_path = (+ Path(self.test_dir) / f"entities_{compression}.parquet"+ )++ exporter.export_entities(self.entities, str(output_path))+ self.assertTrue(output_path.exists())++ # Verify file can be read+ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 2)+ except (ImportError, RuntimeError, OSError) as e:+ if "codec" in str(e).lower() or "compression" in str(e).lower():+ self.skipTest(f"Codec {compression} not available: {e}")+ raise++ def test_export_entities_empty(self):+ """Test exporting empty entities list raises ValidationError."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "empty_entities.parquet"++ with self.assertRaises(ValidationError):+ exporter.export_entities([], str(output_path))++ def test_export_entities_metadata_handling(self):+ """Test entity metadata is correctly serialized."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "entities_metadata.parquet"++ exporter.export_entities(self.entities, str(output_path))++ # Read and verify metadata+ table = pq.read_table(str(output_path))+ metadata_col = table.column("metadata")++ # First entity should have metadata+ first_metadata = metadata_col[0].as_py()+ self.assertIsNotNone(first_metadata)+ self.assertIn("keys", first_metadata)+ self.assertIn("values", first_metadata)+ self.assertEqual(set(first_metadata["keys"]), {"age", "city"})++ def test_export_relationships_basic(self):+ """Test basic relationship export to Parquet."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "relationships.parquet"++ exporter.export_relationships(self.relationships, str(output_path))+ self.assertTrue(output_path.exists())++ # Read and verify Parquet file+ table = pq.read_table(str(output_path))++ # Verify schema+ from semantica.export.parquet_exporter import RELATIONSHIP_SCHEMA++ self.assertEqual(table.schema, RELATIONSHIP_SCHEMA)++ # Verify data+ self.assertEqual(table.num_rows, 2)+ self.assertEqual(table.column("id")[0].as_py(), "r1")+ self.assertEqual(table.column("source_id")[0].as_py(), "e1")+ self.assertEqual(table.column("target_id")[0].as_py(), "e2")+ self.assertEqual(table.column("type")[0].as_py(), "WORKS_FOR")++ def test_export_relationships_field_normalization(self):+ """Test relationship field name normalization."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "relationships_normalized.parquet"++ # Relationships with various field name variations+ varied_rels = [+ {"id": "r1", "source": "e1", "target": "e2", "type": "TYPE1"},+ {"source_id": "e2", "target_id": "e3", "relationship_type": "TYPE2"},+ {"from_id": "e3", "to_id": "e1", "relation_type": "TYPE3"},+ ]++ exporter.export_relationships(varied_rels, str(output_path))+ self.assertTrue(output_path.exists())++ # Read and verify normalization+ table = pq.read_table(str(output_path))++ self.assertEqual(table.num_rows, 3)+ self.assertEqual(table.column("source_id")[0].as_py(), "e1")+ self.assertEqual(table.column("target_id")[0].as_py(), "e2")++ def test_export_relationships_empty(self):+ """Test exporting empty relationships list raises ValidationError."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "empty_relationships.parquet"++ with self.assertRaises(ValidationError):+ exporter.export_relationships([], str(output_path))++ def test_export_relationships_auto_id_generation(self):+ """Test relationship ID is auto-generated when missing."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "relationships_auto_id.parquet"++ # Relationships without IDs+ rels_no_id = [+ {"source_id": "e1", "target_id": "e2", "type": "REL1"},+ {"source_id": "e2", "target_id": "e3", "type": "REL2"},+ ]++ exporter.export_relationships(rels_no_id, str(output_path))++ # Read and verify IDs were generated+ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 2)+ self.assertIsNotNone(table.column("id")[0].as_py())+ self.assertIsNotNone(table.column("id")[1].as_py())++ def test_export_knowledge_graph_basic(self):+ """Test basic knowledge graph export to multiple Parquet files."""+ exporter = ParquetExporter()+ base_path = Path(self.test_dir) / "kg"++ exporter.export_knowledge_graph(self.kg, str(base_path))++ # Verify files were created+ entities_path = Path(self.test_dir) / "kg_entities.parquet"+ rels_path = Path(self.test_dir) / "kg_relationships.parquet"++ self.assertTrue(entities_path.exists())+ self.assertTrue(rels_path.exists())++ # Verify entities+ entities_table = pq.read_table(str(entities_path))+ self.assertEqual(entities_table.num_rows, 2)++ # Verify relationships+ rels_table = pq.read_table(str(rels_path))+ self.assertEqual(rels_table.num_rows, 2)++ def test_export_knowledge_graph_invalid_input(self):+ """Test knowledge graph export with invalid input raises ValidationError."""+ exporter = ParquetExporter()+ base_path = Path(self.test_dir) / "kg_invalid"++ # Not a dictionary+ with self.assertRaises(ValidationError):+ exporter.export_knowledge_graph("not a dict", str(base_path))++ # Missing both entities and relationships+ with self.assertRaises(ValidationError):+ exporter.export_knowledge_graph({"metadata": {}}, str(base_path))++ def test_export_knowledge_graph_partial(self):+ """Test knowledge graph export with only entities or relationships."""+ exporter = ParquetExporter()++ # Only entities+ kg_entities_only = {"entities": self.entities}+ base_path_ent = Path(self.test_dir) / "kg_entities_only"+ exporter.export_knowledge_graph(kg_entities_only, str(base_path_ent))++ entities_path = Path(self.test_dir) / "kg_entities_only_entities.parquet"+ self.assertTrue(entities_path.exists())++ # Only relationships+ kg_rels_only = {"relationships": self.relationships}+ base_path_rel = Path(self.test_dir) / "kg_rels_only"+ exporter.export_knowledge_graph(kg_rels_only, str(base_path_rel))++ rels_path = Path(self.test_dir) / "kg_rels_only_relationships.parquet"+ self.assertTrue(rels_path.exists())++ def test_export_generic_list(self):+ """Test generic export with list of dictionaries."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "generic_list.parquet"++ exporter.export(self.entities, str(output_path), schema=None)+ self.assertTrue(output_path.exists())++ # Verify file can be read (schema auto-selected based on data structure)+ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 2)++ def test_export_generic_dict(self):+ """Test generic export with dictionary (multiple files)."""+ exporter = ParquetExporter()+ base_path = Path(self.test_dir) / "generic_dict"++ data_dict = {"entities": self.entities, "relationships": self.relationships}++ exporter.export(data_dict, str(base_path))++ # Verify both files were created+ entities_path = Path(self.test_dir) / "generic_dict_entities.parquet"+ rels_path = Path(self.test_dir) / "generic_dict_relationships.parquet"++ self.assertTrue(entities_path.exists())+ self.assertTrue(rels_path.exists())++ def test_export_invalid_data_type(self):+ """Test export with invalid data type raises ValidationError."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "invalid.parquet"++ with self.assertRaises(ValidationError):+ exporter.export("invalid string data", str(output_path))++ def test_pandas_compatibility(self):+ """Test exported Parquet files can be read by pandas."""+ try:+ import pandas as pd+ except ImportError:+ self.skipTest("pandas not installed")++ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "pandas_test.parquet"++ exporter.export_entities(self.entities, str(output_path))++ # Read with pandas+ df = pd.read_parquet(str(output_path))++ self.assertEqual(len(df), 2)+ self.assertIn("id", df.columns)+ self.assertIn("text", df.columns)+ self.assertIn("type", df.columns)++ def test_file_size_comparison(self):+ """Test Parquet file sizes with different compression."""+ exporter_snappy = ParquetExporter(compression="snappy")+ exporter_gzip = ParquetExporter(compression="gzip")+ exporter_none = ParquetExporter(compression="none")++ path_snappy = Path(self.test_dir) / "size_snappy.parquet"+ path_gzip = Path(self.test_dir) / "size_gzip.parquet"+ path_none = Path(self.test_dir) / "size_none.parquet"++ # Create larger dataset for meaningful comparison+ large_entities = self.entities * 100++ exporter_snappy.export_entities(large_entities, str(path_snappy))+ exporter_gzip.export_entities(large_entities, str(path_gzip))+ exporter_none.export_entities(large_entities, str(path_none))++ size_snappy = path_snappy.stat().st_size+ size_none = path_none.stat().st_size++ # Uncompressed should be largest+ self.assertGreater(size_none, size_snappy)++ # All files should be readable+ for path in [path_snappy, path_gzip, path_none]:+ table = pq.read_table(str(path))+ self.assertEqual(table.num_rows, 200)++ def test_entity_missing_id_skipped(self):+ """Test entities without IDs are skipped with warning."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "entities_missing_id.parquet"++ # Mix of entities with and without IDs+ entities_mixed = [+ {"id": "e1", "text": "Valid Entity"},+ {"text": "Missing ID"}, # No ID+ {"id": "e2", "text": "Another Valid"},+ ]++ exporter.export_entities(entities_mixed, str(output_path))++ # Only 2 entities should be exported (one without ID is skipped)+ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 2)++ def test_relationship_missing_source_target_skipped(self):+ """Test relationships without source/target are skipped with warning."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "rels_missing.parquet"++ # Mix of valid and invalid relationships+ rels_mixed = [+ {"id": "r1", "source_id": "e1", "target_id": "e2"},+ {"id": "r2", "target_id": "e2"}, # Missing source+ {"id": "r3", "source_id": "e1"}, # Missing target+ {"id": "r4", "source_id": "e3", "target_id": "e4"},+ ]++ exporter.export_relationships(rels_mixed, str(output_path))++ # Only 2 valid relationships should be exported+ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 2)++ def test_all_entities_skipped_raises_error(self):+ """Test that exporting entities with all skipped raises ValidationError."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "all_skipped.parquet"++ # All entities missing IDs+ bad_entities = [+ {"text": "No ID 1"},+ {"text": "No ID 2"},+ "not a dict",+ ]++ with self.assertRaises(ValidationError) as cm:+ exporter.export_entities(bad_entities, str(output_path))++ self.assertIn("No valid entities", str(cm.exception))++ def test_all_relationships_skipped_raises_error(self):+ """Test that exporting relationships with all skipped raises ValidationError."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "all_rels_skipped.parquet"++ # All relationships missing source or target+ bad_rels = [+ {"id": "r1", "source_id": "e1"}, # Missing target+ {"id": "r2", "target_id": "e2"}, # Missing source+ "not a dict",+ ]++ with self.assertRaises(ValidationError) as cm:+ exporter.export_relationships(bad_rels, str(output_path))++ self.assertIn("No valid relationships", str(cm.exception))++ def test_invalid_confidence_values_handled(self):+ """Test that invalid confidence values are handled gracefully."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "invalid_confidence.parquet"++ entities_with_invalid_conf = [+ {"id": "e1", "text": "Valid", "confidence": 0.9},+ {"id": "e2", "text": "String conf", "confidence": "invalid"},+ {"id": "e3", "text": "None conf", "confidence": None},+ ]++ exporter.export_entities(entities_with_invalid_conf, str(output_path))++ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 3)+ # First entity has valid confidence+ self.assertEqual(table.column("confidence")[0].as_py(), 0.9)+ # Second entity has invalid confidence (should be None)+ self.assertIsNone(table.column("confidence")[1].as_py())+ # Third entity has None confidence+ self.assertIsNone(table.column("confidence")[2].as_py())++ def test_invalid_start_end_values_handled(self):+ """Test that invalid start/end offset values are handled gracefully."""+ exporter = ParquetExporter()+ output_path = Path(self.test_dir) / "invalid_offsets.parquet"++ entities_with_invalid_offsets = [+ {"id": "e1", "text": "Valid", "start": 0, "end": 10},+ {"id": "e2", "text": "String offsets", "start": "abc", "end": "def"},+ {"id": "e3", "text": "None offsets", "start": None, "end": None},+ ]++ exporter.export_entities(entities_with_invalid_offsets, str(output_path))++ table = pq.read_table(str(output_path))+ self.assertEqual(table.num_rows, 3)+ # First entity has valid offsets+ self.assertEqual(table.column("start")[0].as_py(), 0)+ self.assertEqual(table.column("end")[0].as_py(), 10)+ # Second entity has invalid offsets (should be None)+ self.assertIsNone(table.column("start")[1].as_py())+ self.assertIsNone(table.column("end")[1].as_py())+ # Third entity has None offsets+ self.assertIsNone(table.column("start")[2].as_py())+ self.assertIsNone(table.column("end")[2].as_py())+++if __name__ == "__main__":+ unittest.main()