Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

fix(mypy): enable pydantic mypy plugin, fix template_enricher/yaml_loader

dextmorgn521138d1dc58
diff --git a/flowsint-api/pyproject.toml b/flowsint-api/pyproject.toml
index f24207d1..3af01b75 100644
--- a/flowsint-api/pyproject.toml
+++ b/flowsint-api/pyproject.toml
@@ -56,6 +56,10 @@ python_version = "3.11"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
+# Without this, mypy doesn't understand Field()'s required/optional
+# semantics and flags every pydantic model construction that omits a
+# field with a default as a "missing named argument" false positive.
+plugins = ["pydantic.mypy"]
# See the matching override's rationale in flowsint-core/pyproject.toml:
# pytest fixture params aren't worth fully annotating for test-only code.
diff --git a/flowsint-core/pyproject.toml b/flowsint-core/pyproject.toml
index 01347865..2d1280c5 100644
--- a/flowsint-core/pyproject.toml
+++ b/flowsint-core/pyproject.toml
@@ -55,6 +55,10 @@ python_version = "3.11"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
+# Without this, mypy doesn't understand Field()'s required/optional
+# semantics and flags every pydantic model construction that omits a
+# field with a default as a "missing named argument" false positive.
+plugins = ["pydantic.mypy"]
# Test fixtures (MockEnricher etc.) use the same InputType/OutputType
# class-attribute pattern real enrichers do — see the matching override
diff --git a/flowsint-core/src/flowsint_core/core/enricher_base.py b/flowsint-core/src/flowsint_core/core/enricher_base.py
index 56d758ba..5e071bc3 100644
--- a/flowsint-core/src/flowsint_core/core/enricher_base.py
+++ b/flowsint-core/src/flowsint_core/core/enricher_base.py
@@ -414,8 +414,12 @@ class Enricher(ABC):
return values
return cleaned
+ # input_data's real element type is whatever the subclass's InputType
+ # is (same as results/InputType/OutputType elsewhere in this class) —
+ # Any here for the same reason, not a str-typed default that happened
+ # to be wrong.
def postprocess(
- self, results: List[Dict[str, Any]], input_data: Optional[List[str]] = None
+ self, results: List[Dict[str, Any]], input_data: Optional[List[Any]] = None
) -> List[Dict[str, Any]]:
return results
diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.py
index 10b7e514..f06f0dbe 100644
--- a/flowsint-core/src/flowsint_core/core/template_enricher.py
+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py
@@ -45,12 +45,13 @@ Example template:
import asyncio
import xml.etree.ElementTree as ET
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Type, cast
import httpx
from flowsint_core.core.enricher_base import Enricher
from flowsint_core.core.logger import Logger
+from flowsint_core.core.vault import VaultProtocol
from flowsint_core.templates.loader.yaml_loader import (
SSRFError,
TemplateRenderError,
@@ -88,9 +89,9 @@ class TemplateEnricher(Enricher):
template: Template,
sketch_id: Optional[str] = None,
scan_id: Optional[str] = None,
- vault=None,
+ vault: Optional[VaultProtocol] = None,
params: Optional[Dict[str, Any]] = None,
- ):
+ ) -> None:
# Build params schema from template secrets
params_schema = self._build_params_schema_from_template(template)
@@ -125,7 +126,10 @@ class TemplateEnricher(Enricher):
def _detect_type(self, input_type: str) -> type[FlowsintType]:
"""Resolve a type name to its FlowsintType class."""
- DetectedType = get_type(input_type)
+ # flowsint_types isn't py.typed, so get_type() resolves to Any here
+ # regardless of its own (correct) declared return type — cast to
+ # what it actually returns rather than losing the check entirely.
+ DetectedType = cast(Optional[Type[FlowsintType]], get_type(input_type))
if not DetectedType:
raise TypeError(f"Type '{input_type}' is not present in registry.")
return DetectedType
@@ -144,7 +148,7 @@ class TemplateEnricher(Enricher):
"""Return formatted markdown documentation for template enrichers."""
return "Template-based enricher. See template definition for details."
- async def async_init(self):
+ async def async_init(self) -> None:
"""Initialize the enricher, resolving vault secrets."""
await super().async_init()
# Store resolved secrets for template rendering
@@ -532,8 +536,11 @@ class TemplateEnricher(Enricher):
return results
- def postprocess(self, results: List[Any], input_data: List[Any] = []) -> List[Any]:
+ def postprocess(
+ self, results: List[Any], input_data: Optional[List[Any]] = None
+ ) -> List[Any]:
"""Log results and return them."""
+ input_data = input_data or []
for input, output in zip(input_data, results):
self.create_node(input)
self.create_node(output)
diff --git a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
index bfd47404..e16e8022 100644
--- a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
+++ b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
@@ -1,6 +1,6 @@
import ipaddress
import re
-from typing import Any, Optional, Set
+from typing import Any, Dict, Optional, Set
from urllib.parse import urlparse
import yaml
@@ -212,7 +212,7 @@ class YamlLoader:
Returns:
New dictionary with all placeholders substituted
"""
- result = {}
+ result: Dict[str, Any] = {}
for key, value in data.items():
if isinstance(value, str):
result[key] = YamlLoader.render_template(value, values, sanitize)
diff --git a/flowsint-core/tests/templates/test_template_enricher.py b/flowsint-core/tests/templates/test_template_enricher.py
index e7da0c7c..f946ddc7 100644
--- a/flowsint-core/tests/templates/test_template_enricher.py
+++ b/flowsint-core/tests/templates/test_template_enricher.py
@@ -30,12 +30,12 @@ def create_test_template(
output_type: str = "Ip",
url: str = "https://api.example.com/{{address}}",
method: str = "GET",
- headers: dict = None,
- params: dict = None,
+ headers: Optional[dict] = None,
+ params: Optional[dict] = None,
body: Optional[str] = None,
- response_map: dict = None,
+ response_map: Optional[dict] = None,
response_expect: str = "json",
- secrets: list = None,
+ secrets: Optional[list] = None,
retry: Optional[TemplateRetryConfig] = None,
is_array: bool = False,
array_path: Optional[str] = None,
@@ -70,7 +70,7 @@ def create_test_template(
class MockVault:
"""Mock vault for testing secret resolution."""
- def __init__(self, secrets: dict = None):
+ def __init__(self, secrets: Optional[dict] = None):
self._secrets = secrets or {}
def get_secret(self, name: str) -> Optional[str]:
diff --git a/flowsint-enrichers/pyproject.toml b/flowsint-enrichers/pyproject.toml
index 387b6bd2..c8750844 100644
--- a/flowsint-enrichers/pyproject.toml
+++ b/flowsint-enrichers/pyproject.toml
@@ -54,6 +54,10 @@ python_version = "3.11"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
+# Without this, mypy doesn't understand Field()'s required/optional
+# semantics and flags every pydantic model construction that omits a
+# field with a default as a "missing named argument" false positive.
+plugins = ["pydantic.mypy"]
# Every enricher declares InputType/OutputType as plain class attributes
# (see Enricher.InputType = NotImplemented in enricher_base.py, overridden
diff --git a/flowsint-types/pyproject.toml b/flowsint-types/pyproject.toml
index feddf1b3..5a6b68e6 100644
--- a/flowsint-types/pyproject.toml
+++ b/flowsint-types/pyproject.toml
@@ -30,6 +30,10 @@ python_version = "3.11"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
+# Without this, mypy doesn't understand Field()'s required/optional
+# semantics and flags every pydantic model construction that omits a
+# field with a default as a "missing named argument" false positive.
+plugins = ["pydantic.mypy"]
# See the matching override's rationale in flowsint-core/pyproject.toml:
# pytest fixture params aren't worth fully annotating for test-only code.
diff --git a/flowsint-types/src/flowsint_types/asn.py b/flowsint-types/src/flowsint_types/asn.py
index 70ea5998..10cad498 100644
--- a/flowsint-types/src/flowsint_types/asn.py
+++ b/flowsint-types/src/flowsint_types/asn.py
@@ -85,7 +85,7 @@ class ASN(FlowsintType):
return self
@classmethod
- def from_string(cls, line: str):
+ def from_string(cls, line: str) -> "ASN":
"""Parse an ASN from a raw string."""
return cls(asn_str=line.strip())