A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
fix(mypy): enable pydantic mypy plugin, fix template_enricher/yaml_loader
dextmorgn521138d1dc58
diff --git a/flowsint-api/pyproject.toml b/flowsint-api/pyproject.tomlindex f24207d1..3af01b75 100644--- a/flowsint-api/pyproject.toml+++ b/flowsint-api/pyproject.toml@@ -56,6 +56,10 @@ python_version = "3.11"warn_return_any = truewarn_unused_configs = truedisallow_untyped_defs = true+# Without this, mypy doesn't understand Field()'s required/optional+# semantics and flags every pydantic model construction that omits a+# field with a default as a "missing named argument" false positive.+plugins = ["pydantic.mypy"]# See the matching override's rationale in flowsint-core/pyproject.toml:# pytest fixture params aren't worth fully annotating for test-only code.diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.tomlindex 01347865..2d1280c5 100644--- a/flowsint-core/pyproject.toml+++ b/flowsint-core/pyproject.toml@@ -55,6 +55,10 @@ python_version = "3.11"warn_return_any = truewarn_unused_configs = truedisallow_untyped_defs = true+# Without this, mypy doesn't understand Field()'s required/optional+# semantics and flags every pydantic model construction that omits a+# field with a default as a "missing named argument" false positive.+plugins = ["pydantic.mypy"]# Test fixtures (MockEnricher etc.) use the same InputType/OutputType# class-attribute pattern real enrichers do — see the matching overridediff --git a/flowsint-core/src/flowsint_core/core/enricher_base.py b/flowsint-core/src/flowsint_core/core/enricher_base.pyindex 56d758ba..5e071bc3 100644--- a/flowsint-core/src/flowsint_core/core/enricher_base.py+++ b/flowsint-core/src/flowsint_core/core/enricher_base.py@@ -414,8 +414,12 @@ class Enricher(ABC):return valuesreturn cleaned+ # input_data's real element type is whatever the subclass's InputType+ # is (same as results/InputType/OutputType elsewhere in this class) —+ # Any here for the same reason, not a str-typed default that happened+ # to be wrong.def postprocess(- self, results: List[Dict[str, Any]], input_data: Optional[List[str]] = None+ self, results: List[Dict[str, Any]], input_data: Optional[List[Any]] = None) -> List[Dict[str, Any]]:return resultsdiff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.pyindex 10b7e514..f06f0dbe 100644--- a/flowsint-core/src/flowsint_core/core/template_enricher.py+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py@@ -45,12 +45,13 @@ Example template:import asyncioimport xml.etree.ElementTree as ET-from typing import Any, Dict, List, Optional+from typing import Any, Dict, List, Optional, Type, castimport httpxfrom flowsint_core.core.enricher_base import Enricherfrom flowsint_core.core.logger import Logger+from flowsint_core.core.vault import VaultProtocolfrom flowsint_core.templates.loader.yaml_loader import (SSRFError,TemplateRenderError,@@ -88,9 +89,9 @@ class TemplateEnricher(Enricher):template: Template,sketch_id: Optional[str] = None,scan_id: Optional[str] = None,- vault=None,+ vault: Optional[VaultProtocol] = None,params: Optional[Dict[str, Any]] = None,- ):+ ) -> None:# Build params schema from template secretsparams_schema = self._build_params_schema_from_template(template)@@ -125,7 +126,10 @@ class TemplateEnricher(Enricher):def _detect_type(self, input_type: str) -> type[FlowsintType]:"""Resolve a type name to its FlowsintType class."""- DetectedType = get_type(input_type)+ # flowsint_types isn't py.typed, so get_type() resolves to Any here+ # regardless of its own (correct) declared return type — cast to+ # what it actually returns rather than losing the check entirely.+ DetectedType = cast(Optional[Type[FlowsintType]], get_type(input_type))if not DetectedType:raise TypeError(f"Type '{input_type}' is not present in registry.")return DetectedType@@ -144,7 +148,7 @@ class TemplateEnricher(Enricher):"""Return formatted markdown documentation for template enrichers."""return "Template-based enricher. See template definition for details."- async def async_init(self):+ async def async_init(self) -> None:"""Initialize the enricher, resolving vault secrets."""await super().async_init()# Store resolved secrets for template rendering@@ -532,8 +536,11 @@ class TemplateEnricher(Enricher):return results- def postprocess(self, results: List[Any], input_data: List[Any] = []) -> List[Any]:+ def postprocess(+ self, results: List[Any], input_data: Optional[List[Any]] = None+ ) -> List[Any]:"""Log results and return them."""+ input_data = input_data or []for input, output in zip(input_data, results):self.create_node(input)self.create_node(output)diff --git a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.pyindex bfd47404..e16e8022 100644--- a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py+++ b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py@@ -1,6 +1,6 @@import ipaddressimport re-from typing import Any, Optional, Set+from typing import Any, Dict, Optional, Setfrom urllib.parse import urlparseimport yaml@@ -212,7 +212,7 @@ class YamlLoader:Returns:New dictionary with all placeholders substituted"""- result = {}+ result: Dict[str, Any] = {}for key, value in data.items():if isinstance(value, str):result[key] = YamlLoader.render_template(value, values, sanitize)diff --git a/flowsint-core/tests/templates/test_template_enricher.py b/flowsint-core/tests/templates/test_template_enricher.pyindex e7da0c7c..f946ddc7 100644--- a/flowsint-core/tests/templates/test_template_enricher.py+++ b/flowsint-core/tests/templates/test_template_enricher.py@@ -30,12 +30,12 @@ def create_test_template(output_type: str = "Ip",url: str = "https://api.example.com/{{address}}",method: str = "GET",- headers: dict = None,- params: dict = None,+ headers: Optional[dict] = None,+ params: Optional[dict] = None,body: Optional[str] = None,- response_map: dict = None,+ response_map: Optional[dict] = None,response_expect: str = "json",- secrets: list = None,+ secrets: Optional[list] = None,retry: Optional[TemplateRetryConfig] = None,is_array: bool = False,array_path: Optional[str] = None,@@ -70,7 +70,7 @@ def create_test_template(class MockVault:"""Mock vault for testing secret resolution."""- def __init__(self, secrets: dict = None):+ def __init__(self, secrets: Optional[dict] = None):self._secrets = secrets or {}def get_secret(self, name: str) -> Optional[str]:diff --git a/flowsint-enrichers/pyproject.toml b/flowsint-enrichers/pyproject.tomlindex 387b6bd2..c8750844 100644--- a/flowsint-enrichers/pyproject.toml+++ b/flowsint-enrichers/pyproject.toml@@ -54,6 +54,10 @@ python_version = "3.11"warn_return_any = truewarn_unused_configs = truedisallow_untyped_defs = true+# Without this, mypy doesn't understand Field()'s required/optional+# semantics and flags every pydantic model construction that omits a+# field with a default as a "missing named argument" false positive.+plugins = ["pydantic.mypy"]# Every enricher declares InputType/OutputType as plain class attributes# (see Enricher.InputType = NotImplemented in enricher_base.py, overriddendiff --git a/flowsint-types/pyproject.toml b/flowsint-types/pyproject.tomlindex feddf1b3..5a6b68e6 100644--- a/flowsint-types/pyproject.toml+++ b/flowsint-types/pyproject.toml@@ -30,6 +30,10 @@ python_version = "3.11"warn_return_any = truewarn_unused_configs = truedisallow_untyped_defs = true+# Without this, mypy doesn't understand Field()'s required/optional+# semantics and flags every pydantic model construction that omits a+# field with a default as a "missing named argument" false positive.+plugins = ["pydantic.mypy"]# See the matching override's rationale in flowsint-core/pyproject.toml:# pytest fixture params aren't worth fully annotating for test-only code.diff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.pyindex 70ea5998..10cad498 100644--- a/flowsint-types/src/flowsint_types/asn.py+++ b/flowsint-types/src/flowsint_types/asn.py@@ -85,7 +85,7 @@ class ASN(FlowsintType):return self@classmethod- def from_string(cls, line: str):+ def from_string(cls, line: str) -> "ASN":"""Parse an ASN from a raw string."""return cls(asn_str=line.strip())