A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): template enricher v1
dextmorgn327123a7298d
diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.pynew file mode 100644index 00000000..01d61df1--- /dev/null+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py@@ -0,0 +1,116 @@+"""+This is the base class for the enricher from a template.+This enricher takes a yaml template as input, that describes the expected behaviour.+For now, this is only limited to straightforward http GET requests, with minimal params/payloads.++It does support usage of third party api keys, using the Vault system, just like any other enricher.+"""++from typing import Any, Dict, List, Optional++import requests+from flowsint_enrichers.registry import flowsint_enricher+from flowsint_types import FlowsintType, get_type++from flowsint_core.core.enricher_base import Enricher+from flowsint_core.core.logger import Logger+from flowsint_core.templates.loader.yaml_loader import YamlLoader+from flowsint_core.templates.types import Template+++class TemplateEnricher(Enricher):+ """Resolve domain names to IP addresses."""++ InputType = FlowsintType+ OutputType = FlowsintType++ def __init__(+ self,+ template: Template,+ sketch_id: Optional[str] = None,+ scan_id: Optional[str] = None,+ vault=None,+ params: Optional[Dict[str, Any]] = None,+ ):+ super().__init__(+ sketch_id=sketch_id,+ scan_id=scan_id,+ vault=vault,+ params=params,+ )+ self.template = template+ self.params_schema = self.get_params_schema()+ self.InputType = self._detect_type(self.template.input.type)+ self.OutputType = self._detect_type(self.template.output.type)+ self.request = self.template.request++ def _detect_type(self, inputType: str) -> type[FlowsintType]:+ DetectedType = get_type(inputType)+ if not DetectedType:+ raise TypeError(f"Type {inputType} is not present in registry.")+ return DetectedType++ def name(self) -> str: # type: ignore[override]+ return self.template.name++ def category(self) -> str: # type: ignore[override]+ return self.template.category++ def key(self) -> str: # type: ignore[override]+ return self.template.input.key++ @classmethod+ def documentation(cls) -> str:+ """Return formatted markdown documentation for the domain resolver enricher."""+ return ""++ def _build_mapped_result(self, result: dict) -> Any:+ mappings = self.template.response.map++ # build object from mapping, as raw dict+ output_dict = {}+ for key, value in mappings.items():+ output_dict[key] = result.get(value)++ return self.OutputType(**output_dict)++ async def scan(self, values: List[Any]) -> List[Any]:+ results: List[Any] = []+ req = self.request+ # the key used to match the formatted string+ key = self.template.input.key++ if not key:+ raise Exception(+ f"Key is missing for intput type {self.template.input.type}."+ )++ for d in values:+ url = YamlLoader.render_template(req.url, {key: getattr(d, key)})+ print(url)+ try:+ resp = requests.get(+ url,+ params=req.params,+ timeout=10,+ )+ resp.raise_for_status()+ data = resp.json()++ results.append(self._build_mapped_result(data))+ except Exception as e:+ Logger.info(+ self.sketch_id,+ {"message": f"Error resolving {req.url}: {e}"},+ )+ continue+ return results++ def postprocess(self, results: List[Any], input_data: List[Any] = []) -> List[Any]:+ for res in results:+ self.log_graph_message(f"Resolved {self.request.url} -> {res.model_dump()}")+ return results+++InputType = TemplateEnricher.InputType+OutputType = TemplateEnricher.OutputTypediff --git a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.pyindex 325539f9..c8e17bc9 100644--- a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py+++ b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py@@ -7,6 +7,7 @@ from flowsint_types import TYPE_REGISTRYfrom flowsint_core.templates.types import TemplateTEMPLATE_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")+ALLOWED_METHODS = ["GET"]class YamlLoader:@@ -25,17 +26,25 @@ class YamlLoader:input: dict | None = raw.get("input", None)if not input or not isinstance(input, dict):- raise TypeError("Missing 'input' property in the yaml.")+ raise ValueError("Missing 'input' property in the yaml.")input_type: str | None = input.get("type", None)if not input_type:- raise TypeError("Missing 'input_type' property in the yaml.")+ raise ValueError("Missing 'input_type' property in the yaml.")DetectedType = TYPE_REGISTRY.get(input_type)+ request = raw.get("request", {})+ method = request.get("method", None)++ if method not in ALLOWED_METHODS:+ raise ValueError(+ f"Method {method} not present in {', '.join(ALLOWED_METHODS)}"+ )+if not DetectedType:- raise TypeError(f"Type '{input_type}' not found in registry.")+ raise ValueError(f"Type '{input_type}' not found in registry.")return Template(**raw)diff --git a/flowsint-core/src/flowsint_core/templates/types.py b/flowsint-core/src/flowsint_core/templates/types.pyindex bfeaab30..391ddd36 100644--- a/flowsint-core/src/flowsint_core/templates/types.py+++ b/flowsint-core/src/flowsint_core/templates/types.py@@ -11,6 +11,12 @@ class TemplateInput(BaseModel):)+class TemplateOutput(BaseModel):+ type: str = Field(+ ..., description="Flowsint Type that the template should return as an output."+ )++class TemplateHttpRequestHeader(BaseModel):name: str = Field(min_length=1, max_length=256, pattern=r"^[A-Za-z0-9\-]+$")value: str = Field(min_length=1, max_length=4096)@@ -83,3 +89,7 @@ class Template(BaseModel):response: TemplateHttpResponse = Field(..., description="Response model for the HTTP response to expect.")+ output: TemplateOutput = Field(+ ...,+ description="Output type of the template.",+ )diff --git a/flowsint-core/tests/templates/enricher.py b/flowsint-core/tests/templates/enricher.pynew file mode 100644index 00000000..fda602f7--- /dev/null+++ b/flowsint-core/tests/templates/enricher.py@@ -0,0 +1,23 @@+from pathlib import Path++import pytest++from flowsint_core.core.template_enricher import TemplateEnricher+from flowsint_core.templates.loader.yaml_loader import YamlLoader+from flowsint_core.templates.types import Template++TEST_DIR = Path(__file__).parent+++async def test_enricher_params():+ template = YamlLoader.get_template_from_file(str(TEST_DIR / "example.yaml"))++ assert isinstance(template, Template)+ assert template.name == "ip-api-lookup"+ assert template.category == "Ip"+ enricher = TemplateEnricher(sketch_id="123", scan_id="123", template=template)+ pre = enricher.preprocess(["8.8.8.8"])+ scans = await enricher.scan(pre)+ print(scans)+ assert enricherdiff --git a/flowsint-core/tests/templates/example-invalid.yaml b/flowsint-core/tests/templates/example-invalid.yamlnew file mode 100644index 00000000..913891c3--- /dev/null+++ b/flowsint-core/tests/templates/example-invalid.yaml@@ -0,0 +1,24 @@+name: shodan_ip_lookup+category: Ip+version: 1.0+type: enricher+input:+ type: Ip+ key: address+request:+ method: DELETE+ url: http://ip-api.com/json/{{address}}+ headers:+ Authorization: "Bearer {{secrets.SHODAN_API_KEY}}"++output:+ type: Ip++response:+ expect: json+ map:+ lat: Ip.latitute+ lon: Ip.longitude+ country: Ip.country+ city: Ip.city+ isp: Ip.ispdiff --git a/flowsint-core/tests/templates/example.yaml b/flowsint-core/tests/templates/example.yamlindex 44e2e70e..1adc909b 100644--- a/flowsint-core/tests/templates/example.yaml+++ b/flowsint-core/tests/templates/example.yaml@@ -1,4 +1,4 @@-name: shodan_ip_lookup+name: ip-api-lookupcategory: Ipversion: 1.0type: enricher@@ -8,17 +8,16 @@ input:request:method: GETurl: http://ip-api.com/json/{{address}}- headers:- Authorization: "Bearer {{secrets.SHODAN_API_KEY}}"output:type: Ipresponse:expect: json- map:- lat: Ip.latitute- lon: Ip.longitude- country: Ip.country- city: Ip.city- isp: Ip.isp+ map: # Map received keys (right) to object attributes (left, ex: Ip.latitude, etc)+ address: query+ latitute: lat+ longitude: lon+ country: country+ city: city+ isp: ispdiff --git a/flowsint-core/tests/templates/loader.py b/flowsint-core/tests/templates/loader.pyindex e25ba6d8..339e13a5 100644--- a/flowsint-core/tests/templates/loader.py+++ b/flowsint-core/tests/templates/loader.py@@ -1,13 +1,16 @@import json+from pathlib import Path++import pytestfrom flowsint_core.templates.loader.yaml_loader import YamlLoaderfrom flowsint_core.templates.types import Template+TEST_DIR = Path(__file__).parent+def test_yaml_loader():- file = YamlLoader.get_template_from_file(- "/Users/eliottmorcillo/Documents/Perso/projets/reconurge/flowsint/flowsint-core/tests/templates/example.yaml"- )+ file = YamlLoader.get_template_from_file(str(TEST_DIR / "example.yaml"))assert isinstance(file, Template)assert file.name == "shodan_ip_lookup"assert file.category == "Ip"@@ -21,3 +24,21 @@ def test_render_template():url = YamlLoader.render_template(url_template, {"address": "8.8.8.8"})assert url == "http://ip-api.com/json/8.8.8.8"+++def test_render_template_2():+ url_template = "http://ip-api.com/json?ip={{address}}&domain={{domain}}"++ url = YamlLoader.render_template(+ url_template, {"address": "8.8.8.8", "domain": "mydomain.com"}+ )++ assert url == "http://ip-api.com/json?ip=8.8.8.8&domain=mydomain.com"+ print(url)+++def test_invalid_method():+ with pytest.raises(ValueError) as exc_info:+ YamlLoader.get_template_from_file(str(TEST_DIR / "example-invalid.yaml"))++ assert "not present in" in str(exc_info.value).lower()