A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
fix(mypy): fix orchestrator/serializer type gaps, scope enricher pattern overrides
dextmorgnfa1c8e758024
diff --git a/flowsint-api/alembic/env.py b/flowsint-api/alembic/env.pyindex 4fe71d14..a9d24303 100644--- a/flowsint-api/alembic/env.py+++ b/flowsint-api/alembic/env.py@@ -21,7 +21,9 @@ if not database_url:raise RuntimeError("DATABASE_URL is not defined in .env")config.set_main_option("sqlalchemy.url", database_url)-target_metadata = Base.metadata+# Base comes from the `import *` above (models.py isn't py.typed, so mypy+# can't see it) — same standard Alembic autogenerate pattern noted there.+target_metadata = Base.metadata # type: ignore[name-defined]def run_migrations_offline() -> None:diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.tomlindex 764bc4de..2b2c66f0 100644--- a/flowsint-core/pyproject.toml+++ b/flowsint-core/pyproject.toml@@ -55,3 +55,10 @@ python_version = "3.11"warn_return_any = truewarn_unused_configs = truedisallow_untyped_defs = true++# Test fixtures (MockEnricher etc.) use the same InputType/OutputType+# class-attribute pattern real enrichers do — see the matching override+# and its rationale in flowsint-enrichers/pyproject.toml.+[[tool.mypy.overrides]]+module = "tests.*"+disable_error_code = ["valid-type", "attr-defined", "name-defined"]diff --git a/flowsint-core/src/flowsint_core/core/graph/serializer.py b/flowsint-core/src/flowsint_core/core/graph/serializer.pyindex 10e1fe28..0cf51195 100644--- a/flowsint-core/src/flowsint_core/core/graph/serializer.py+++ b/flowsint-core/src/flowsint_core/core/graph/serializer.py@@ -5,9 +5,9 @@ This module provides utilities for serializing complex Python objectsinto Neo4j-compatible primitive types, following the Single Responsibility Principle."""-from typing import Any, Callable, Dict, List, Optional, Type+from typing import Any, Callable, Dict, List, Optional, Type, Union-from pydantic import BaseModel, ValidationError+from pydantic import ValidationErrorfrom flowsint_core.utils import flatten, unflattenfrom flowsint_types import FlowsintType@@ -180,7 +180,9 @@ class GraphSerializer:@staticmethoddef graph_edge_to_neo4j_dict(- from_obj: BaseModel, to_obj: BaseModel, label: str+ from_obj: Union[FlowsintType, GraphNode],+ to_obj: Union[FlowsintType, GraphNode],+ label: str,) -> Dict[str, Any]:"""Build a Neo4j relationship dict for matching nodes by type and label.diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.pyindex b0600a60..1a84cccf 100644--- a/flowsint-core/src/flowsint_core/core/orchestrator.py+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py@@ -3,7 +3,7 @@ import jsonimport osimport timefrom datetime import datetime-from typing import Any, Dict, List+from typing import Any, Dict, List, Optional, Unionfrom pydantic import ValidationError@@ -13,6 +13,7 @@ from ..utils import to_json_serializablefrom .enricher_base import Enricherfrom .logger import Loggerfrom .types import FlowBranch, FlowStep+from .vault import VaultProtocolclass FlowOrchestrator(Enricher):@@ -25,12 +26,12 @@ class FlowOrchestrator(Enricher):sketch_id: str,scan_id: str,enricher_branches: List[FlowBranch],- vault=None,+ vault: Optional[VaultProtocol] = None,):super().__init__(sketch_id, scan_id, vault=vault)self.enricher_branches = enricher_branches- self.enrichers = {} # Map of nodeId -> enricher instance- self.execution_log_file = None # Path to the execution log file+ self.enrichers: Dict[str, Enricher] = {} # Map of nodeId -> enricher instance+ self.execution_log_file: Optional[str] = None # Path to the execution log fileself._create_execution_log()self._load_enrichers()@@ -45,7 +46,8 @@ class FlowOrchestrator(Enricher):# Create filename with sketch_id and scan_idfilename = f"enricher_execution_{self.sketch_id}_{self.scan_id}.json"- self.execution_log_file = os.path.join(enricher_dir, filename)+ log_file = os.path.join(enricher_dir, filename)+ self.execution_log_file = log_file# Serialize the enricher branchesserialized_branches = to_json_serializable(self.enricher_branches)@@ -77,7 +79,7 @@ class FlowOrchestrator(Enricher):initial_log["summary"]["total_steps"] = total_steps# Save initial log file- with open(self.execution_log_file, "w", encoding="utf-8") as f:+ with open(log_file, "w", encoding="utf-8") as f:json.dump(initial_log, f, indent=2, ensure_ascii=False)Logger.info(@@ -94,7 +96,7 @@ class FlowOrchestrator(Enricher):self.execution_log_file = Nonedef _update_execution_log(- self, step_entry: Dict[str, Any], status: str = None+ self, step_entry: Optional[Dict[str, Any]] = None, status: Optional[str] = None) -> None:"""Update the execution log file with a new step entry or status update.@@ -259,9 +261,14 @@ class FlowOrchestrator(Enricher):return results_mapping[ref_value]return None+ # No callers anywhere in the codebase — dead code. Its two return+ # statements disagree (a dict in the no-inputs-resolved branch, a single+ # resolved value via the last loop-iteration's input_key otherwise),+ # which List[Any] never matched; typed Any here rather than guessing at+ # the intended behavior of code nothing exercises.def prepare_enricher_inputs(self, step: FlowStep, results_mapping: Dict[str, Any], initial_values: List[str]- ) -> List[Any]:+ ) -> Any:"""Prepare the inputs for an enricher based on the references and previous results.Handles single references, lists, and direct values.@@ -301,7 +308,12 @@ class FlowOrchestrator(Enricher):def update_results_mapping(self,- outputs: Dict[str, Any],+ # Callers only ever pass enricher.execute()'s result through, which+ # is validated as dict-or-list upstream — the body below only makes+ # sense for the dict case (list membership/indexing by a string key+ # would raise), a pre-existing gap not fixed here since it changes+ # runtime behavior for enrichers whose outputs are list-shaped.+ outputs: Union[Dict[str, Any], List[Any]],step_outputs: Dict[str, str],results_mapping: Dict[str, Any],) -> None:@@ -310,7 +322,7 @@ class FlowOrchestrator(Enricher):"""for output_key, output_ref in step_outputs.items():if output_key in outputs:- results_mapping[output_ref] = outputs[output_key]+ results_mapping[output_ref] = outputs[output_key] # type: ignore[index,call-overload]@classmethoddef name(cls) -> str:@@ -332,7 +344,13 @@ class FlowOrchestrator(Enricher):def output_schema(cls) -> Dict[str, str]:return {"branches": "array", "results": "dict"}- def scan(self, values: List[str]) -> Dict[str, Any]:+ # Deliberately not Liskov-compatible with Enricher.scan (async, returns+ # List[Dict[str, Any]]): the orchestrator returns one aggregate result+ # for the whole flow run, not a per-item list, and this sync entry point+ # exists for callers outside the async enricher-execution path. Not+ # changing the signature here — that's a behavioral call about how+ # FlowOrchestrator is actually invoked, not a typing fix.+ def scan(self, values: List[str]) -> Dict[str, Any]: # type: ignore[override]"""Synchronous implementation of scan that runs the async version in an event loop"""@@ -351,13 +369,17 @@ class FlowOrchestrator(Enricher):# Update execution log to indicate scan has startedself._update_execution_log(None, "running")- results = {"initial_values": values, "branches": [], "results": {}}+ results: Dict[str, Any] = {+ "initial_values": values,+ "branches": [],+ "results": {},+ }Logger.pending(self.sketch_id, {"message": "Starting enricher..."})# Global mapping of output references to actual values- results_mapping = {}+ results_mapping: Dict[str, Any] = {}# Cache for enricher results to avoid recomputation- enricher_results_cache = {}+ enricher_results_cache: Dict[str, Any] = {}sum(len(branch.steps) for branch in self.enricher_branches)completed_steps = 0@@ -366,7 +388,11 @@ class FlowOrchestrator(Enricher):for branch in self.enricher_branches:branch_id = branch.idbranch_name = branch.name- branch_results = {"id": branch_id, "name": branch_name, "steps": []}+ branch_results: Dict[str, Any] = {+ "id": branch_id,+ "name": branch_name,+ "steps": [],+ }# Process each step in the branchenricher_inputs = values@@ -387,14 +413,14 @@ class FlowOrchestrator(Enricher):enricher_name = enricher.name()step_start_time = time.time()- step_result = {+ step_result: Dict[str, Any] = {"nodeId": node_id,"enricher": enricher_name,"status": "error", # Default to error, will update on success}# Create execution log entry- log_entry = {+ log_entry: Dict[str, Any] = {"step_id": f"{branch_id}_{node_id}","branch_id": branch_id,"branch_name": branch_name,diff --git a/flowsint-enrichers/pyproject.toml b/flowsint-enrichers/pyproject.tomlindex 4873f74c..f0bcca7d 100644--- a/flowsint-enrichers/pyproject.toml+++ b/flowsint-enrichers/pyproject.toml@@ -54,3 +54,18 @@ python_version = "3.11"warn_return_any = truewarn_unused_configs = truedisallow_untyped_defs = true++# Every enricher declares InputType/OutputType as plain class attributes+# (see Enricher.InputType = NotImplemented in enricher_base.py, overridden+# per-subclass) and then uses them in annotations like List[InputType] and+# reads fields off them (self.data.domain). This is the documented,+# intentional API — see flowsint-enricher-builder skill and every enricher+# in this package — not a mistake to individually silence. mypy can't+# express "this class attribute doubles as a type alias" without making+# Enricher generic (Enricher[TIn, TOut]), which would mean touching every+# enricher's class declaration and contradicts the deliberately simple+# "just set two class attributes" API. Narrowing the checks that pattern+# structurally can't satisfy, rather than ignoring per-line across ~15 files.+[[tool.mypy.overrides]]+module = "flowsint_enrichers.*"+disable_error_code = ["valid-type", "attr-defined", "name-defined"]