Skip to content

Graph-Native Infrastructure for Context and Accountable AI Systems

feat: add Apache Airflow ingestor (#1584)

David Millerfdf3fd8ca341
diff --git a/docs/docs.json b/docs/docs.json
index 4c9aa69c..2c5da4c3 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -109,6 +109,7 @@
"integrations/databricks",
"integrations/bigquery",
"integrations/salesforce",
+ "integrations/airflow",
"integrations/redshift",
"integrations/powerbi"
]
diff --git a/docs/integrations/airflow.md b/docs/integrations/airflow.md
new file mode 100644
index 00000000..9c58cd94
--- /dev/null
+++ b/docs/integrations/airflow.md
@@ -0,0 +1,209 @@
+---
+title: 'Apache Airflow Integration'
+description: 'Ingest DAG, task, and dependency metadata from Apache Airflow into Semantica.'
+icon: 'wind'
+---
+
+> Ingest Apache Airflow DAGs, tasks, and task dependencies through the stable Airflow REST API for pipeline and lineage analysis in Semantica.
+
+## Installation
+
+```bash
+pip install "semantica[ingest-airflow]"
+```
+
+## Basic Usage
+
+```python
+from semantica.ingest import AirflowIngestor
+
+ingestor = AirflowIngestor(
+ base_url="https://airflow.example.com",
+ token="your-access-token",
+)
+
+data = ingestor.ingest()
+documents = ingestor.export_as_documents(data)
+
+print(f"DAGs: {len(data.dags)}")
+print(f"Tasks: {len(data.tasks)}")
+print(f"Dependencies: {len(data.dependencies)}")
+
+ingestor.close()
+```
+
+`base_url` must point to the Airflow webserver root.
+
+Do not include `/api/v1` in `base_url`; the connector adds the stable REST API path automatically.
+
+## Authentication
+
+### Bearer Token
+
+```python
+from semantica.ingest import AirflowIngestor
+
+ingestor = AirflowIngestor(
+ base_url="https://airflow.example.com",
+ token="your-access-token",
+)
+```
+
+### Username and Password
+
+```python
+from semantica.ingest import AirflowIngestor
+
+ingestor = AirflowIngestor(
+ base_url="https://airflow.example.com",
+ username="airflow-user",
+ password="your-password",
+)
+```
+
+Username and password must be supplied together.
+
+## Filtering DAGs
+
+Restrict ingestion to selected DAG IDs:
+
+```python
+data = ingestor.ingest(
+ dag_ids=["daily_etl", "warehouse_sync"],
+)
+```
+
+Exclude paused DAGs:
+
+```python
+data = ingestor.ingest(
+ include_paused=False,
+)
+```
+
+You can combine both options:
+
+```python
+data = ingestor.ingest(
+ dag_ids=["daily_etl", "warehouse_sync"],
+ include_paused=False,
+)
+```
+
+## Pipeline Lineage
+
+The connector derives task-level dependencies from each Airflow task's `downstream_task_ids`.
+
+For example:
+
+```python
+[
+ {
+ "dag_id": "daily_etl",
+ "upstream_task_id": "extract",
+ "downstream_task_id": "transform",
+ },
+ {
+ "dag_id": "daily_etl",
+ "upstream_task_id": "transform",
+ "downstream_task_id": "load",
+ },
+]
+```
+
+Duplicate dependency edges are removed automatically.
+
+## Exported Documents
+
+`export_as_documents()` converts ingested Airflow metadata into GraphBuilder-friendly dictionaries.
+
+The connector exports three document types:
+
+- `airflow_dag`
+- `airflow_task`
+- `airflow_dependency`
+
+Generated identifiers use these forms:
+
+```text
+airflow:dag:<dag_id>
+airflow:task:<dag_id>:<task_id>
+airflow:dependency:<dag_id>:<upstream_task_id>:<downstream_task_id>
+```
+
+## Private Airflow Deployments
+
+All requests to the user-provided Airflow endpoint pass through Semantica's SSRF protection.
+
+Private IP addresses are blocked by default.
+
+For a trusted Airflow deployment on a private network:
+
+```python
+from semantica.ingest import AirflowIngestor
+
+ingestor = AirflowIngestor(
+ base_url="http://10.0.0.20:8080",
+ username="airflow-user",
+ password="your-password",
+ allow_private_ips=True,
+)
+```
+
+Only enable `allow_private_ips=True` for endpoints you trust.
+
+## Pagination
+
+DAG results are retrieved using Airflow's `limit` and `offset` pagination.
+
+The default page size is 100.
+
+```python
+data = ingestor.ingest(
+ page_limit=100,
+)
+```
+
+A different positive page size can be supplied when required.
+
+## Error Handling
+
+The connector raises Semantica errors for common failures, including:
+
+- missing or invalid connection configuration
+- unsuccessful Airflow API requests
+- invalid JSON responses
+- unexpected API response structures
+- invalid pagination limits
+
+## API Coverage
+
+The connector currently uses these stable Airflow REST API endpoints:
+
+```text
+GET /api/v1/dags
+GET /api/v1/dags/{dag_id}/tasks
+```
+
+Task dependencies are derived from each task's `downstream_task_ids`.
+
+## Read-Only Behaviour
+
+The Airflow connector performs metadata ingestion only.
+
+It does not:
+
+- create DAGs
+- modify DAGs
+- trigger DAG runs
+- pause or unpause DAGs
+- modify tasks
+- delete DAGs or tasks
+
+## Closing the Connection
+
+Close the underlying HTTP session when ingestion is complete:
+
+```python
+ingestor.close()
+```
diff --git a/pyproject.toml b/pyproject.toml
index 1ae3caa2..2b6ab28d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -131,6 +131,7 @@ db-bigquery = ["google-cloud-bigquery>=3.0.0"]
ingest-parquet = ["pyarrow>=24.0.0"]
ingest-arrow = ["pyarrow>=24.0.0"]
ingest-sap = ["requests>=2.28.0"]
+ingest-airflow = ["requests>=2.28.0"]
ingest-git = ["GitPython>=3.1.58"]
db-all = [
diff --git a/semantica/ingest/__init__.py b/semantica/ingest/__init__.py
index 80b26836..93ad67a9 100644
--- a/semantica/ingest/__init__.py
+++ b/semantica/ingest/__init__.py
@@ -240,6 +240,10 @@ _LAZY_EXPORTS: Dict[str, Tuple[str, str]] = {
"SAPIngestor": (".sap_ingestor", "SAPIngestor"),
"SAPODataEntity": (".sap_ingestor", "SAPODataEntity"),
"SAPODataConnector": (".sap_ingestor", "SAPODataConnector"),
+ # Apache Airflow ingestion
+ "AirflowIngestor": (".airflow_ingestor", "AirflowIngestor"),
+ "AirflowData": (".airflow_ingestor", "AirflowData"),
+ "AirflowConnector": (".airflow_ingestor", "AirflowConnector"),
# Databricks ingestion
"DatabricksIngestor": (".databricks_ingestor", "DatabricksIngestor"),
"DatabricksData": (".databricks_ingestor", "DatabricksData"),
@@ -311,6 +315,11 @@ _OPTIONAL_DEPENDENCY_MESSAGES = {
"Salesforce ingestion requires optional dependency 'simple-salesforce'. "
"Install it with: pip install 'semantica[db-salesforce]'"
),
+ ".airflow_ingestor": (
+ "Apache Airflow ingestion requires optional dependency 'requests'. "
+ "Install it with: "
+ "pip install \"semantica[ingest-airflow]\""
+ ),
".redshift_ingestor": (
"Redshift ingestion requires optional dependency 'redshift-connector'. "
"Install it with: pip install 'semantica[db-redshift]'"
@@ -478,6 +487,10 @@ __all__ = [
"SAPIngestor",
"SAPODataEntity",
"SAPODataConnector",
+ # Apache Airflow ingestion
+ "AirflowIngestor",
+ "AirflowData",
+ "AirflowConnector",
# Databricks ingestion
"DatabricksIngestor",
"DatabricksData",
diff --git a/semantica/ingest/airflow_ingestor.py b/semantica/ingest/airflow_ingestor.py
new file mode 100644
index 00000000..8216938c
--- /dev/null
+++ b/semantica/ingest/airflow_ingestor.py
@@ -0,0 +1,453 @@
+"""Apache Airflow metadata ingestor.
+
+Provides read-only ingestion of DAG, task, and task-dependency metadata from
+the stable Apache Airflow REST API.
+
+The connector follows Semantica's ingestion pattern:
+
+- ``AirflowData`` stores fetched DAG/task/lineage metadata.
+- ``AirflowConnector`` manages authentication and HTTP transport.
+- ``AirflowIngestor`` fetches metadata and exports
+ GraphBuilder-ready documents.
+
+All outbound requests to the user-supplied Airflow endpoint go through
+``request_with_ssrf_guard``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+from urllib.parse import quote, urljoin
+
+import requests
+
+from ..utils.exceptions import ProcessingError, ValidationError
+from ..utils.logging import get_logger
+from .ssrf import request_with_ssrf_guard
+
+__all__ = [
+ "AirflowData",
+ "AirflowConnector",
+ "AirflowIngestor",
+]
+
+
+_logger = get_logger("airflow_ingestor")
+
+
+@dataclass
+class AirflowData:
+ """Container for Airflow DAG, task, and dependency metadata."""
+
+ dags: List[Dict[str, Any]]
+ tasks: List[Dict[str, Any]]
+ dependencies: List[Dict[str, str]]
+ source: str
+ ingested_at: datetime = field(default_factory=datetime.now)
+
+ def to_documents(self) -> List[Dict[str, Any]]:
+ """Convert Airflow metadata into GraphBuilder-friendly document dicts."""
+
+ documents: List[Dict[str, Any]] = []
+ containment_documents: List[Dict[str, Any]] = []
+ exported_dag_ids = set()
+
+ for dag in self.dags:
+ dag_id = str(dag.get("dag_id") or "")
+ if not dag_id:
+ continue
+
+ doc = dict(dag)
+ doc.setdefault("id", f"airflow:dag:{dag_id}")
+ doc.setdefault("name", dag_id)
+ doc.setdefault("type", "airflow_dag")
+ doc.setdefault("source", self.source)
+ documents.append(doc)
+ exported_dag_ids.add(dag_id)
+
+ for task in self.tasks:
+ dag_id = str(task.get("dag_id") or "")
+ task_id = str(task.get("task_id") or "")
+
+ if not dag_id or not task_id:
+ continue
+
+ task_entity_id = f"airflow:task:{dag_id}:{task_id}"
+ doc = dict(task)
+ doc.setdefault("id", task_entity_id)
+ doc.setdefault("name", task_id)
+ doc.setdefault("type", "airflow_task")
+ doc.setdefault("source", self.source)
+ documents.append(doc)
+
+ if dag_id in exported_dag_ids:
+ containment_documents.append(
+ {
+ "id": f"airflow:contains:{dag_id}:{task_id}",
+ "name": f"{dag_id} contains {task_id}",
+ "type": "airflow_contains",
+ "source": f"airflow:dag:{dag_id}",
+ "target": task_entity_id,
+ "dag_id": dag_id,
+ "task_id": task_id,
+ "metadata": {
+ "airflow_source": self.source,
+ },
+ }
+ )
+
+ documents.extend(containment_documents)
+
+ for dependency in self.dependencies:
+ dag_id = str(dependency.get("dag_id") or "")
+ upstream = str(dependency.get("upstream_task_id") or "")
+ downstream = str(dependency.get("downstream_task_id") or "")
+
+ if not dag_id or not upstream or not downstream:
+ continue
+
+ upstream_id = f"airflow:task:{dag_id}:{upstream}"
+ downstream_id = f"airflow:task:{dag_id}:{downstream}"
+
+ documents.append(
+ {
+ "id": (f"airflow:dependency:{dag_id}:" f"{upstream}:{downstream}"),
+ "name": f"{upstream} -> {downstream}",
+ "type": "airflow_dependency",
+ "source": upstream_id,
+ "target": downstream_id,
+ "dag_id": dag_id,
+ "upstream_task_id": upstream,
+ "downstream_task_id": downstream,
+ "metadata": {
+ "airflow_source": self.source,
+ },
+ }
+ )
+
+ return documents
+
+
+class AirflowConnector:
+ """Connection and authentication management for the Airflow REST API."""
+
+ def __init__(
+ self,
+ base_url: str,
+ *,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+ token: Optional[str] = None,
+ allow_private_ips: bool = False,
+ timeout: int = 30,
+ ) -> None:
+ """Initialize an Airflow REST API connection."""
+
+ if not base_url:
+ raise ValidationError("Airflow base_url is required.")
+
+ self.base_url = base_url.rstrip("/") + "/"
+ self.allow_private_ips = allow_private_ips
+ self.timeout = timeout
+ self.session = requests.Session()
+
+ self.session.headers.update(
+ {
+ "Accept": "application/json",
+ }
+ )
+
+ if token:
+ self.session.headers["Authorization"] = f"Bearer {token}"
+ elif username is not None or password is not None:
+ if not username or password is None:
+ raise ValidationError(
+ "Airflow username and password must be provided together."
+ )
+ self.session.auth = (username, password)
+
+ def get_session(self) -> requests.Session:
+ """Return the configured requests session."""
+
+ return self.session
+
+ def api_url(self, path: str) -> str:
+ """Build a URL below the stable Airflow ``/api/v1/`` namespace."""
+
+ return urljoin(
+ self.base_url,
+ f"api/v1/{path.lstrip('/')}",
+ )
+
+ def request(
+ self,
+ method: str,
+ path: str,
+ **kwargs: Any,
+ ) -> requests.Response:
+ """Perform an SSRF-guarded request against the Airflow API."""
+
+ url = self.api_url(path)
+ kwargs.setdefault("timeout", self.timeout)
+
+ try:
+ response = request_with_ssrf_guard(
+ method,
+ url,
+ session=self.session,
+ allow_private_ips=self.allow_private_ips,
+ allow_private_ips_on_redirect=False,
+ **kwargs,
+ )
+ response.raise_for_status()
+ except requests.exceptions.RequestException as exc:
+ raise ProcessingError(
+ f"Airflow API request failed for '{url}': {exc}"
+ ) from exc
+
+ return response
+
+ def get_json(
+ self,
+ path: str,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ """Perform a GET request and return the JSON object response."""
+
+ response = self.request("GET", path, **kwargs)
+
+ try:
+ payload = response.json()
+ except ValueError as exc:
+ raise ProcessingError(
+ f"Airflow API endpoint '{path}' did not return valid JSON."
+ ) from exc
+
+ if not isinstance(payload, dict):
+ raise ProcessingError(
+ f"Airflow API endpoint '{path}' returned "
+ f"{type(payload).__name__}, expected an object."
+ )
+
+ return payload
+
+ def close(self) -> None:
+ """Close the underlying requests session."""
+
+ self.session.close()
+
+
+class AirflowIngestor:
+ """Read DAG, task, and task-dependency metadata from Apache Airflow."""
+
+ def __init__(
+ self,
+ connector: Optional[AirflowConnector] = None,
+ *,
+ base_url: Optional[str] = None,
+ username: Optional[str] = None,
+ password: Optional[str] = None,
+ token: Optional[str] = None,
+ allow_private_ips: bool = False,
+ timeout: int = 30,
+ ) -> None:
+ """Initialize the Airflow metadata ingestor."""
+
+ if connector is not None:
+ self.connector = connector
+ else:
+ if not base_url:
+ raise ValidationError("Provide either an AirflowConnector or base_url.")
+
+ self.connector = AirflowConnector(
+ base_url,
+ username=username,
+ password=password,
+ token=token,
+ allow_private_ips=allow_private_ips,
+ timeout=timeout,
+ )
+
+ self.logger = _logger
+
+ def list_dags(
+ self,
+ *,
+ limit: int = 100,
+ ) -> List[Dict[str, Any]]:
+ """Fetch all DAG metadata using Airflow REST API pagination."""
+
+ if limit <= 0:
+ raise ValidationError("Airflow DAG page limit must be greater than zero.")
+
+ dags: List[Dict[str, Any]] = []
+ offset = 0
+
+ while True:
+ payload = self.connector.get_json(
+ "dags",
+ params={
+ "limit": limit,
+ "offset": offset,
+ },
+ )
+
+ page = payload.get("dags", [])
+
+ if not isinstance(page, list):
+ raise ProcessingError(
+ "Airflow DAG response field 'dags' is not a list."
+ )
+
+ for dag in page:
+ if isinstance(dag, dict):
+ dags.append(dict(dag))
+
+ total_entries = payload.get("total_entries")
+
+ if not page:
+ break
+
+ offset += len(page)
+
+ if isinstance(total_entries, int):
+ if offset >= total_entries:
+ break
+ elif len(page) < limit:
+ break
+
+ return dags
+
+ def list_tasks(
+ self,
+ dag_id: str,
+ ) -> List[Dict[str, Any]]:
+ """Fetch task metadata for one Airflow DAG."""
+
+ if not dag_id:
+ raise ValidationError("Airflow dag_id is required.")
+
+ encoded_dag_id = quote(dag_id, safe="")
+
+ payload = self.connector.get_json(f"dags/{encoded_dag_id}/tasks")
+
+ raw_tasks = payload.get("tasks", [])
+
+ if not isinstance(raw_tasks, list):
+ raise ProcessingError(
+ f"Airflow task response for DAG '{dag_id}' "
+ "contains a non-list 'tasks' field."
+ )
+
+ tasks: List[Dict[str, Any]] = []
+
+ for raw_task in raw_tasks:
+ if not isinstance(raw_task, dict):
+ continue
+
+ task = dict(raw_task)
+ task["dag_id"] = dag_id
+ tasks.append(task)
+
+ return tasks
+
+ @staticmethod
+ def _dependencies_from_tasks(
+ tasks: List[Dict[str, Any]],
+ ) -> List[Dict[str, str]]:
+ """Derive task lineage edges from ``downstream_task_ids``."""
+
+ dependencies: List[Dict[str, str]] = []
+ seen = set()
+
+ for task in tasks:
+ dag_id = str(task.get("dag_id") or "")
+ task_id = str(task.get("task_id") or "")
+
+ downstream_task_ids = task.get("downstream_task_ids") or []
+
+ if not dag_id or not task_id or not isinstance(downstream_task_ids, list):
+ continue
+
+ for downstream_task_id in downstream_task_ids:
+ downstream = str(downstream_task_id or "")
+
+ if not downstream:
+ continue
+
+ key = (dag_id, task_id, downstream)
+
+ if key in seen:
+ continue
+
+ seen.add(key)
+
+ dependencies.append(
+ {
+ "dag_id": dag_id,
+ "upstream_task_id": task_id,
+ "downstream_task_id": downstream,
+ }
+ )
+
+ return dependencies
+
+ def ingest(
+ self,
+ *,
+ dag_ids: Optional[List[str]] = None,
+ include_paused: bool = True,
+ page_limit: int = 100,
+ ) -> AirflowData:
+ """Fetch DAGs, tasks, and dependency metadata from Airflow."""
+
+ dags = self.list_dags(limit=page_limit)
+
+ if dag_ids is not None:
+ requested = set(dag_ids)
+ dags = [dag for dag in dags if str(dag.get("dag_id") or "") in requested]
+
+ if not include_paused:
+ dags = [dag for dag in dags if not bool(dag.get("is_paused"))]
+
+ tasks: List[Dict[str, Any]] = []
+
+ for dag in dags:
+ dag_id = str(dag.get("dag_id") or "")
+
+ if not dag_id:
+ continue
+
+ try:
+ tasks.extend(self.list_tasks(dag_id))
+ except ProcessingError as exc:
+ self.logger.error(
+ "Failed to fetch Airflow tasks for DAG %s: %s",
+ dag_id,
+ exc,
+ )
+ raise
+
+ dependencies = self._dependencies_from_tasks(tasks)
+
+ return AirflowData(
+ dags=dags,
+ tasks=tasks,
+ dependencies=dependencies,
+ source=self.connector.base_url,
+ )
+
+ def export_as_documents(
+ self,
+ data: AirflowData,
+ ) -> List[Dict[str, Any]]:
+ """Convert ingested Airflow metadata into document dictionaries."""
+
+ return data.to_documents()
+
+ def close(self) -> None:
+ """Close the underlying Airflow connector."""
+
+ self.connector.close()
diff --git a/tests/test_airflow_ingestor.py b/tests/test_airflow_ingestor.py
new file mode 100644
index 00000000..15b0e869
--- /dev/null
+++ b/tests/test_airflow_ingestor.py
@@ -0,0 +1,715 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+import requests
+
+import semantica.ingest as ingest_package
+from semantica.ingest.airflow_ingestor import (
+ AirflowConnector,
+ AirflowData,
+ AirflowIngestor,
+)
+from semantica.kg import GraphBuilder
+from semantica.utils.exceptions import ProcessingError, ValidationError
+
+
+def make_response(
+ payload=None,
+ *,
+ status_code=200,
+ json_error=None,
+):
+ response = MagicMock(spec=requests.Response)
+ response.status_code = status_code
+
+ if json_error is not None:
+ response.json.side_effect = json_error
+ else:
+ response.json.return_value = payload
+
+ response.raise_for_status.return_value = None
+ return response
+
+
+def test_airflow_classes_are_public_package_exports():
+ expected = {
+ "AirflowIngestor": AirflowIngestor,
+ "AirflowData": AirflowData,
+ "AirflowConnector": AirflowConnector,
+ }
+
+ for name, expected_class in expected.items():
+ assert name in ingest_package.__all__
+ assert getattr(ingest_package, name) is expected_class
+
+
+class TestAirflowConnector:
+ def test_requires_base_url(self):
+ with pytest.raises(ValidationError, match="base_url is required"):
+ AirflowConnector("")
+
+ def test_basic_auth_configuration(self):
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ username="user",
+ password="secret",
+ )
+
+ assert connector.session.auth == ("user", "secret")
+ assert connector.base_url == "https://airflow.example.com/"
+
+ connector.close()
+
+ def test_username_requires_password(self):
+ with pytest.raises(
+ ValidationError,
+ match="username and password must be provided together",
+ ):
+ AirflowConnector(
+ "https://airflow.example.com",
+ username="user",
+ )
+
+ def test_bearer_token_configuration(self):
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ token="token-value",
+ )
+
+ assert connector.session.headers["Authorization"] == "Bearer token-value"
+
+ connector.close()
+
+ def test_api_url_uses_stable_v1_namespace(self):
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ )
+
+ assert connector.api_url("dags") == "https://airflow.example.com/api/v1/dags"
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_request_uses_ssrf_guard(self, mock_guard):
+ response = make_response({"dags": []})
+ mock_guard.return_value = response
+
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ token="abc",
+ timeout=15,
+ )
+
+ result = connector.request(
+ "GET",
+ "dags",
+ params={"limit": 10},
+ )
+
+ assert result is response
+
+ mock_guard.assert_called_once_with(
+ "GET",
+ "https://airflow.example.com/api/v1/dags",
+ session=connector.session,
+ allow_private_ips=False,
+ allow_private_ips_on_redirect=False,
+ params={"limit": 10},
+ timeout=15,
+ )
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_request_allows_private_origin_but_blocks_private_redirects(
+ self,
+ mock_guard,
+ ):
+ response = make_response({"dags": []})
+ mock_guard.return_value = response
+
+ connector = AirflowConnector(
+ "http://10.0.0.20:8080",
+ allow_private_ips=True,
+ )
+
+ connector.request("GET", "dags")
+
+ mock_guard.assert_called_once_with(
+ "GET",
+ "http://10.0.0.20:8080/api/v1/dags",
+ session=connector.session,
+ allow_private_ips=True,
+ allow_private_ips_on_redirect=False,
+ timeout=30,
+ )
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_request_converts_http_error(self, mock_guard):
+ response = make_response()
+ response.raise_for_status.side_effect = requests.exceptions.HTTPError(
+ "500 Server Error"
+ )
+ mock_guard.return_value = response
+
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ )
+
+ with pytest.raises(
+ ProcessingError,
+ match="Airflow API request failed",
+ ):
+ connector.request("GET", "dags")
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_request_converts_timeout_error(self, mock_guard):
+ mock_guard.side_effect = requests.exceptions.Timeout("timed out")
+
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ )
+
+ with pytest.raises(
+ ProcessingError,
+ match="Airflow API request failed",
+ ) as exc_info:
+ connector.request("GET", "dags")
+
+ assert isinstance(
+ exc_info.value.__cause__,
+ requests.exceptions.Timeout,
+ )
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_request_converts_connection_error(self, mock_guard):
+ mock_guard.side_effect = requests.exceptions.ConnectionError(
+ "connection failed"
+ )
+
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ )
+
+ with pytest.raises(
+ ProcessingError,
+ match="Airflow API request failed",
+ ) as exc_info:
+ connector.request("GET", "dags")
+
+ assert isinstance(
+ exc_info.value.__cause__,
+ requests.exceptions.ConnectionError,
+ )
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_get_json_rejects_invalid_json(self, mock_guard):
+ mock_guard.return_value = make_response(
+ json_error=ValueError("bad json"),
+ )
+
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ )
+
+ with pytest.raises(
+ ProcessingError,
+ match="did not return valid JSON",
+ ):
+ connector.get_json("dags")
+
+ connector.close()
+
+ @patch("semantica.ingest.airflow_ingestor.request_with_ssrf_guard")
+ def test_get_json_requires_object_response(self, mock_guard):
+ mock_guard.return_value = make_response(["not", "an", "object"])
+
+ connector = AirflowConnector(
+ "https://airflow.example.com",
+ )
+
+ with pytest.raises(
+ ProcessingError,
+ match="expected an object",
+ ):
+ connector.get_json("dags")
+
+ connector.close()
+
+
+class TestAirflowIngestor:
+ def test_requires_connector_or_base_url(self):
+ with pytest.raises(
+ ValidationError,
+ match="Provide either an AirflowConnector or base_url",
+ ):
+ AirflowIngestor()
+
+ def test_list_dags_paginates(self):
+ connector = MagicMock(spec=AirflowConnector)
+
+ connector.get_json.side_effect = [
+ {
+ "dags": [
+ {"dag_id": "dag_one"},
+ {"dag_id": "dag_two"},
+ ],
+ "total_entries": 3,
+ },
+ {
+ "dags": [
+ {"dag_id": "dag_three"},
+ ],
+ "total_entries": 3,
+ },
+ ]
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ dags = ingestor.list_dags(limit=2)
+
+ assert [dag["dag_id"] for dag in dags] == [
+ "dag_one",
+ "dag_two",
+ "dag_three",
+ ]
+
+ assert connector.get_json.call_count == 2
+
+ connector.get_json.assert_any_call(
+ "dags",
+ params={
+ "limit": 2,
+ "offset": 0,
+ },
+ )
+ connector.get_json.assert_any_call(
+ "dags",
+ params={
+ "limit": 2,
+ "offset": 2,
+ },
+ )
+
+ def test_list_dags_continues_when_server_caps_page_size(self):
+ connector = MagicMock(spec=AirflowConnector)
+
+ connector.get_json.side_effect = [
+ {
+ "dags": [
+ {"dag_id": "dag_one"},
+ {"dag_id": "dag_two"},
+ ],
+ "total_entries": 4,
+ },
+ {
+ "dags": [
+ {"dag_id": "dag_three"},
+ {"dag_id": "dag_four"},
+ ],
+ "total_entries": 4,
+ },
+ ]
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ dags = ingestor.list_dags(limit=100)
+
+ assert [dag["dag_id"] for dag in dags] == [
+ "dag_one",
+ "dag_two",
+ "dag_three",
+ "dag_four",
+ ]
+
+ assert connector.get_json.call_count == 2
+ connector.get_json.assert_any_call(
+ "dags",
+ params={
+ "limit": 100,
+ "offset": 0,
+ },
+ )
+ connector.get_json.assert_any_call(
+ "dags",
+ params={
+ "limit": 100,
+ "offset": 2,
+ },
+ )
+
+ def test_list_dags_stops_on_short_page_without_total_entries(self):
+ connector = MagicMock(spec=AirflowConnector)
+
+ connector.get_json.return_value = {
+ "dags": [
+ {"dag_id": "dag_one"},
+ ],
+ }
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ dags = ingestor.list_dags(limit=100)
+
+ assert dags == [{"dag_id": "dag_one"}]
+ assert connector.get_json.call_count == 1
+
+ def test_list_dags_rejects_invalid_limit(self):
+ connector = MagicMock(spec=AirflowConnector)
+ ingestor = AirflowIngestor(connector=connector)
+
+ with pytest.raises(
+ ValidationError,
+ match="page limit must be greater than zero",
+ ):
+ ingestor.list_dags(limit=0)
+
+ def test_list_dags_rejects_non_list_payload(self):
+ connector = MagicMock(spec=AirflowConnector)
+ connector.get_json.return_value = {
+ "dags": "not-a-list",
+ }
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ with pytest.raises(
+ ProcessingError,
+ match="field 'dags' is not a list",
+ ):
+ ingestor.list_dags()
+
+ def test_list_tasks_encodes_dag_id(self):
+ connector = MagicMock(spec=AirflowConnector)
+ connector.get_json.return_value = {
+ "tasks": [
+ {
+ "task_id": "extract",
+ "downstream_task_ids": ["load"],
+ }
+ ]
+ }
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ tasks = ingestor.list_tasks("sales/daily")
+
+ connector.get_json.assert_called_once_with("dags/sales%2Fdaily/tasks")
+
+ assert tasks == [
+ {
+ "task_id": "extract",
+ "downstream_task_ids": ["load"],
+ "dag_id": "sales/daily",
+ }
+ ]
+
+ def test_list_tasks_requires_dag_id(self):
+ connector = MagicMock(spec=AirflowConnector)
+ ingestor = AirflowIngestor(connector=connector)
+
+ with pytest.raises(
+ ValidationError,
+ match="dag_id is required",
+ ):
+ ingestor.list_tasks("")
+
+ def test_list_tasks_rejects_non_list_payload(self):
+ connector = MagicMock(spec=AirflowConnector)
+ connector.get_json.return_value = {
+ "tasks": "not-a-list",
+ }
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ with pytest.raises(
+ ProcessingError,
+ match="non-list 'tasks' field",
+ ):
+ ingestor.list_tasks("example")
+
+ def test_dependencies_are_derived_from_tasks(self):
+ tasks = [
+ {
+ "dag_id": "pipeline",
+ "task_id": "extract",
+ "downstream_task_ids": ["transform"],
+ },
+ {
+ "dag_id": "pipeline",
+ "task_id": "transform",
+ "downstream_task_ids": ["load"],
+ },
+ ]
+
+ dependencies = AirflowIngestor._dependencies_from_tasks(tasks)
+
+ assert dependencies == [
+ {
+ "dag_id": "pipeline",
+ "upstream_task_id": "extract",
+ "downstream_task_id": "transform",
+ },
+ {
+ "dag_id": "pipeline",
+ "upstream_task_id": "transform",
+ "downstream_task_id": "load",
+ },
+ ]
+
+ def test_dependencies_remove_duplicates(self):
+ tasks = [
+ {
+ "dag_id": "pipeline",
+ "task_id": "extract",
+ "downstream_task_ids": [
+ "load",
+ "load",
+ ],
+ }
+ ]
+
+ dependencies = AirflowIngestor._dependencies_from_tasks(tasks)
+
+ assert dependencies == [
+ {
+ "dag_id": "pipeline",
+ "upstream_task_id": "extract",
+ "downstream_task_id": "load",
+ }
+ ]
+
+ def test_ingest_filters_dags_and_paused_dags(self):
+ connector = MagicMock(spec=AirflowConnector)
+ connector.base_url = "https://airflow.example.com/"
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ ingestor.list_dags = MagicMock(
+ return_value=[
+ {
+ "dag_id": "active",
+ "is_paused": False,
+ },
+ {
+ "dag_id": "paused",
+ "is_paused": True,
+ },
+ {
+ "dag_id": "other",
+ "is_paused": False,
+ },
+ ]
+ )
+
+ ingestor.list_tasks = MagicMock(
+ return_value=[
+ {
+ "task_id": "task_a",
+ "downstream_task_ids": [],
+ "dag_id": "active",
+ }
+ ]
+ )
+
+ result = ingestor.ingest(
+ dag_ids=["active", "paused"],
+ include_paused=False,
+ )
+
+ assert [dag["dag_id"] for dag in result.dags] == ["active"]
+ ingestor.list_tasks.assert_called_once_with("active")
+
+ def test_ingest_builds_lineage(self):
+ connector = MagicMock(spec=AirflowConnector)
+ connector.base_url = "https://airflow.example.com/"
+
+ ingestor = AirflowIngestor(connector=connector)
+
+ ingestor.list_dags = MagicMock(
+ return_value=[
+ {
+ "dag_id": "pipeline",
+ "is_paused": False,
+ }
+ ]
+ )
+
+ ingestor.list_tasks = MagicMock(
+ return_value=[
+ {
+ "dag_id": "pipeline",
+ "task_id": "extract",
+ "downstream_task_ids": ["load"],
+ },
+ {
+ "dag_id": "pipeline",
+ "task_id": "load",
+ "downstream_task_ids": [],
+ },
+ ]
+ )
+
+ result = ingestor.ingest()
+
+ assert result.dependencies == [
+ {
+ "dag_id": "pipeline",
+ "upstream_task_id": "extract",
+ "downstream_task_id": "load",
+ }
+ ]
+
+ assert result.source == "https://airflow.example.com/"
+
+ def test_export_as_documents_delegates_to_data(self):
+ connector = MagicMock(spec=AirflowConnector)
+ ingestor = AirflowIngestor(connector=connector)
+
+ data = MagicMock(spec=AirflowData)
+ data.to_documents.return_value = [{"id": "document"}]
+
+ documents = ingestor.export_as_documents(data)
+
+ assert documents == [{"id": "document"}]
+ data.to_documents.assert_called_once_with()
+
+ def test_close_closes_connector(self):
+ connector = MagicMock(spec=AirflowConnector)
+ ingestor = AirflowIngestor(connector=connector)
+
+ ingestor.close()
+
+ connector.close.assert_called_once_with()
+
+
+class TestAirflowData:
+ def test_to_documents_exports_dags_tasks_and_dependencies(self):
+ data = AirflowData(
+ dags=[
+ {
+ "dag_id": "pipeline",
+ "description": "Example pipeline",
+ }
+ ],
+ tasks=[
+ {
+ "dag_id": "pipeline",
+ "task_id": "extract",
+ },
+ {
+ "dag_id": "pipeline",
+ "task_id": "load",
+ },
+ ],
+ dependencies=[
+ {
+ "dag_id": "pipeline",
+ "upstream_task_id": "extract",
+ "downstream_task_id": "load",
+ }
+ ],
+ source="https://airflow.example.com/",
+ )
+
+ documents = data.to_documents()
+
+ assert len(documents) == 6
+
+ assert documents[0]["id"] == "airflow:dag:pipeline"
+ assert documents[0]["type"] == "airflow_dag"
+
+ assert documents[1]["id"] == "airflow:task:pipeline:extract"
+ assert documents[1]["type"] == "airflow_task"
+
+ assert documents[2]["id"] == "airflow:task:pipeline:load"
+ assert documents[2]["type"] == "airflow_task"
+
+ containments = documents[3:5]
+ assert containments == [
+ {
+ "id": "airflow:contains:pipeline:extract",
+ "name": "pipeline contains extract",
+ "type": "airflow_contains",
+ "source": "airflow:dag:pipeline",
+ "target": "airflow:task:pipeline:extract",
+ "dag_id": "pipeline",
+ "task_id": "extract",
+ "metadata": {
+ "airflow_source": "https://airflow.example.com/",
+ },
+ },
+ {
+ "id": "airflow:contains:pipeline:load",
+ "name": "pipeline contains load",
+ "type": "airflow_contains",
+ "source": "airflow:dag:pipeline",
+ "target": "airflow:task:pipeline:load",
+ "dag_id": "pipeline",
+ "task_id": "load",
+ "metadata": {
+ "airflow_source": "https://airflow.example.com/",
+ },
+ },
+ ]
+
+ dependency = documents[5]
+
+ assert dependency["id"] == "airflow:dependency:pipeline:extract:load"
+ assert dependency["type"] == "airflow_dependency"
+ assert dependency["source"] == "airflow:task:pipeline:extract"
+ assert dependency["target"] == "airflow:task:pipeline:load"
+ assert dependency["metadata"] == {
+ "airflow_source": "https://airflow.example.com/",
+ }
+
+ def test_dependency_documents_build_graph_relationships(self):
+ data = AirflowData(
+ dags=[],
+ tasks=[
+ {
+ "dag_id": "pipeline",
+ "task_id": "extract",
+ },
+ {
+ "dag_id": "pipeline",
+ "task_id": "load",
+ },
+ ],
+ dependencies=[
+ {
+ "dag_id": "pipeline",
+ "upstream_task_id": "extract",
+ "downstream_task_id": "load",
+ }
+ ],
+ source="https://airflow.example.com/",
+ )
+
+ documents = data.to_documents()
+
+ builder = GraphBuilder()
+ entities = []
+ relationships = []
+
+ for document in documents:
+ builder._process_item(
+ document,
+ entities,
+ relationships,
+ )
+
+ assert len(relationships) == 1
+
+ relationship = relationships[0]
+
+ assert relationship["source"] == "airflow:task:pipeline:extract"
+ assert relationship["target"] == "airflow:task:pipeline:load"
+ assert relationship["type"] == "airflow_dependency"
diff --git a/uv.lock b/uv.lock
index 6055963b..77a1be50 100644
--- a/uv.lock
+++ b/uv.lock
@@ -8812,6 +8812,9 @@ infra = [
{ name = "pulsar-client" },
{ name = "redis" },
]
+ingest-airflow = [
+ { name = "requests" },
+]
ingest-arrow = [
{ name = "pyarrow" },
]
@@ -9067,6 +9070,7 @@ requires-dist = [
{ name = "redis", marker = "extra == 'infra'", specifier = ">=4.3.0" },
{ name = "redshift-connector", marker = "extra == 'db-redshift'", specifier = ">=2.0.0" },
{ name = "requests", specifier = ">=2.34.2" },
+ { name = "requests", marker = "extra == 'ingest-airflow'", specifier = ">=2.28.0" },
{ name = "requests", marker = "extra == 'ingest-sap'", specifier = ">=2.28.0" },
{ name = "rich", specifier = ">=12.5.0" },
{ name = "scikit-learn", specifier = ">=1.7.2" },
@@ -9099,7 +9103,7 @@ requires-dist = [
{ name = "weaviate-client", marker = "extra == 'vectorstore-weaviate'", specifier = ">=4.0.0" },
{ name = "websockets", marker = "extra == 'explorer'", specifier = ">=15.0.1" },
]
-provides-extras = ["llm-openai", "llm-groq", "llm-gemini", "llm-anthropic", "llm-ollama", "llm-deepseek", "llm-novita", "llm-litellm", "llm-instructor", "llm-all", "documents", "parse-docling", "parse-pdf", "shacl", "db-snowflake", "db-databricks", "db-arrow", "db-salesforce", "db-redshift", "db-bigquery", "ingest-parquet", "ingest-arrow", "ingest-sap", "ingest-git", "db-all", "models-huggingface", "embeddings-local", "nlp-spacy", "graph-neo4j", "graph-falkordb", "graph-amazon-neptune", "graph-apache-age", "graph-embeddings", "graph-all", "tripletstore-oxigraph", "vectorstore-faiss", "vectorstore-qdrant", "vectorstore-weaviate", "vectorstore-pinecone", "vectorstore-milvus", "vectorstore-pgvector", "vectorstore-sqlite", "vectorstore-all", "infra", "cloud", "monitoring", "viz", "media", "gpu", "agno", "crewai", "langchain", "google-adk", "open-ontologies", "watch", "split-tiktoken", "split-community", "split-topic", "split-all", "explorer", "explorer-lite", "all"]
+provides-extras = ["llm-openai", "llm-groq", "llm-gemini", "llm-anthropic", "llm-ollama", "llm-deepseek", "llm-novita", "llm-litellm", "llm-instructor", "llm-all", "documents", "parse-docling", "parse-pdf", "shacl", "db-snowflake", "db-databricks", "db-arrow", "db-salesforce", "db-redshift", "db-bigquery", "ingest-parquet", "ingest-arrow", "ingest-sap", "ingest-airflow", "ingest-git", "db-all", "models-huggingface", "embeddings-local", "nlp-spacy", "graph-neo4j", "graph-falkordb", "graph-amazon-neptune", "graph-apache-age", "graph-embeddings", "graph-all", "tripletstore-oxigraph", "vectorstore-faiss", "vectorstore-qdrant", "vectorstore-weaviate", "vectorstore-pinecone", "vectorstore-milvus", "vectorstore-pgvector", "vectorstore-sqlite", "vectorstore-all", "infra", "cloud", "monitoring", "viz", "media", "gpu", "agno", "crewai", "langchain", "google-adk", "open-ontologies", "watch", "split-tiktoken", "split-community", "split-topic", "split-all", "explorer", "explorer-lite", "all"]
[package.metadata.requires-dev]
dev = [