Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat(core): template enricher v1

dextmorgn327123a7298d
diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.py
new file mode 100644
index 00000000..01d61df1
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py
@@ -0,0 +1,116 @@
+"""
+This is the base class for the enricher from a template.
+This enricher takes a yaml template as input, that describes the expected behaviour.
+For now, this is only limited to straightforward http GET requests, with minimal params/payloads.
+
+It does support usage of third party api keys, using the Vault system, just like any other enricher.
+"""
+
+from typing import Any, Dict, List, Optional
+
+import requests
+from flowsint_enrichers.registry import flowsint_enricher
+from flowsint_types import FlowsintType, get_type
+
+from flowsint_core.core.enricher_base import Enricher
+from flowsint_core.core.logger import Logger
+from flowsint_core.templates.loader.yaml_loader import YamlLoader
+from flowsint_core.templates.types import Template
+
+
+class TemplateEnricher(Enricher):
+ """Resolve domain names to IP addresses."""
+
+ InputType = FlowsintType
+ OutputType = FlowsintType
+
+ def __init__(
+ self,
+ template: Template,
+ sketch_id: Optional[str] = None,
+ scan_id: Optional[str] = None,
+ vault=None,
+ params: Optional[Dict[str, Any]] = None,
+ ):
+ super().__init__(
+ sketch_id=sketch_id,
+ scan_id=scan_id,
+ vault=vault,
+ params=params,
+ )
+ self.template = template
+ self.params_schema = self.get_params_schema()
+ self.InputType = self._detect_type(self.template.input.type)
+ self.OutputType = self._detect_type(self.template.output.type)
+ self.request = self.template.request
+
+ def _detect_type(self, inputType: str) -> type[FlowsintType]:
+ DetectedType = get_type(inputType)
+ if not DetectedType:
+ raise TypeError(f"Type {inputType} is not present in registry.")
+ return DetectedType
+
+ def name(self) -> str: # type: ignore[override]
+ return self.template.name
+
+ def category(self) -> str: # type: ignore[override]
+ return self.template.category
+
+ def key(self) -> str: # type: ignore[override]
+ return self.template.input.key
+
+ @classmethod
+ def documentation(cls) -> str:
+ """Return formatted markdown documentation for the domain resolver enricher."""
+ return ""
+
+ def _build_mapped_result(self, result: dict) -> Any:
+ mappings = self.template.response.map
+
+ # build object from mapping, as raw dict
+ output_dict = {}
+ for key, value in mappings.items():
+ output_dict[key] = result.get(value)
+
+ return self.OutputType(**output_dict)
+
+ async def scan(self, values: List[Any]) -> List[Any]:
+ results: List[Any] = []
+ req = self.request
+ # the key used to match the formatted string
+ key = self.template.input.key
+
+ if not key:
+ raise Exception(
+ f"Key is missing for intput type {self.template.input.type}."
+ )
+
+ for d in values:
+ url = YamlLoader.render_template(req.url, {key: getattr(d, key)})
+ print(url)
+ try:
+ resp = requests.get(
+ url,
+ params=req.params,
+ timeout=10,
+ )
+ resp.raise_for_status()
+ data = resp.json()
+
+ results.append(self._build_mapped_result(data))
+ except Exception as e:
+ Logger.info(
+ self.sketch_id,
+ {"message": f"Error resolving {req.url}: {e}"},
+ )
+ continue
+ return results
+
+ def postprocess(self, results: List[Any], input_data: List[Any] = []) -> List[Any]:
+ for res in results:
+ self.log_graph_message(f"Resolved {self.request.url} -> {res.model_dump()}")
+ return results
+
+
+InputType = TemplateEnricher.InputType
+OutputType = TemplateEnricher.OutputType
diff --git a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
index 325539f9..c8e17bc9 100644
--- a/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
+++ b/flowsint-core/src/flowsint_core/templates/loader/yaml_loader.py
@@ -7,6 +7,7 @@ from flowsint_types import TYPE_REGISTRY
from flowsint_core.templates.types import Template
TEMPLATE_RE = re.compile(r"\{\{\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*\}\}")
+ALLOWED_METHODS = ["GET"]
class YamlLoader:
@@ -25,17 +26,25 @@ class YamlLoader:
input: dict | None = raw.get("input", None)
if not input or not isinstance(input, dict):
- raise TypeError("Missing 'input' property in the yaml.")
+ raise ValueError("Missing 'input' property in the yaml.")
input_type: str | None = input.get("type", None)
if not input_type:
- raise TypeError("Missing 'input_type' property in the yaml.")
+ raise ValueError("Missing 'input_type' property in the yaml.")
DetectedType = TYPE_REGISTRY.get(input_type)
+ request = raw.get("request", {})
+ method = request.get("method", None)
+
+ if method not in ALLOWED_METHODS:
+ raise ValueError(
+ f"Method {method} not present in {', '.join(ALLOWED_METHODS)}"
+ )
+
if not DetectedType:
- raise TypeError(f"Type '{input_type}' not found in registry.")
+ raise ValueError(f"Type '{input_type}' not found in registry.")
return Template(**raw)
diff --git a/flowsint-core/src/flowsint_core/templates/types.py b/flowsint-core/src/flowsint_core/templates/types.py
index bfeaab30..391ddd36 100644
--- a/flowsint-core/src/flowsint_core/templates/types.py
+++ b/flowsint-core/src/flowsint_core/templates/types.py
@@ -11,6 +11,12 @@ class TemplateInput(BaseModel):
)
+class TemplateOutput(BaseModel):
+ type: str = Field(
+ ..., description="Flowsint Type that the template should return as an output."
+ )
+
+
class TemplateHttpRequestHeader(BaseModel):
name: str = Field(min_length=1, max_length=256, pattern=r"^[A-Za-z0-9\-]+$")
value: str = Field(min_length=1, max_length=4096)
@@ -83,3 +89,7 @@ class Template(BaseModel):
response: TemplateHttpResponse = Field(
..., description="Response model for the HTTP response to expect."
)
+ output: TemplateOutput = Field(
+ ...,
+ description="Output type of the template.",
+ )
diff --git a/flowsint-core/tests/templates/enricher.py b/flowsint-core/tests/templates/enricher.py
new file mode 100644
index 00000000..fda602f7
--- /dev/null
+++ b/flowsint-core/tests/templates/enricher.py
@@ -0,0 +1,23 @@
+from pathlib import Path
+
+import pytest
+
+from flowsint_core.core.template_enricher import TemplateEnricher
+from flowsint_core.templates.loader.yaml_loader import YamlLoader
+from flowsint_core.templates.types import Template
+
+TEST_DIR = Path(__file__).parent
+
+
+async def test_enricher_params():
+ template = YamlLoader.get_template_from_file(str(TEST_DIR / "example.yaml"))
+
+ assert isinstance(template, Template)
+ assert template.name == "ip-api-lookup"
+ assert template.category == "Ip"
+ enricher = TemplateEnricher(sketch_id="123", scan_id="123", template=template)
+ pre = enricher.preprocess(["8.8.8.8"])
+ scans = await enricher.scan(pre)
+ print(scans)
+ assert enricher
diff --git a/flowsint-core/tests/templates/example-invalid.yaml b/flowsint-core/tests/templates/example-invalid.yaml
new file mode 100644
index 00000000..913891c3
--- /dev/null
+++ b/flowsint-core/tests/templates/example-invalid.yaml
@@ -0,0 +1,24 @@
+name: shodan_ip_lookup
+category: Ip
+version: 1.0
+type: enricher
+input:
+ type: Ip
+ key: address
+request:
+ method: DELETE
+ url: http://ip-api.com/json/{{address}}
+ headers:
+ Authorization: "Bearer {{secrets.SHODAN_API_KEY}}"
+
+output:
+ type: Ip
+
+response:
+ expect: json
+ map:
+ lat: Ip.latitute
+ lon: Ip.longitude
+ country: Ip.country
+ city: Ip.city
+ isp: Ip.isp
diff --git a/flowsint-core/tests/templates/example.yaml b/flowsint-core/tests/templates/example.yaml
index 44e2e70e..1adc909b 100644
--- a/flowsint-core/tests/templates/example.yaml
+++ b/flowsint-core/tests/templates/example.yaml
@@ -1,4 +1,4 @@
-name: shodan_ip_lookup
+name: ip-api-lookup
category: Ip
version: 1.0
type: enricher
@@ -8,17 +8,16 @@ input:
request:
method: GET
url: http://ip-api.com/json/{{address}}
- headers:
- Authorization: "Bearer {{secrets.SHODAN_API_KEY}}"
output:
type: Ip
response:
expect: json
- map:
- lat: Ip.latitute
- lon: Ip.longitude
- country: Ip.country
- city: Ip.city
- isp: Ip.isp
+ map: # Map received keys (right) to object attributes (left, ex: Ip.latitude, etc)
+ address: query
+ latitute: lat
+ longitude: lon
+ country: country
+ city: city
+ isp: isp
diff --git a/flowsint-core/tests/templates/loader.py b/flowsint-core/tests/templates/loader.py
index e25ba6d8..339e13a5 100644
--- a/flowsint-core/tests/templates/loader.py
+++ b/flowsint-core/tests/templates/loader.py
@@ -1,13 +1,16 @@
import json
+from pathlib import Path
+
+import pytest
from flowsint_core.templates.loader.yaml_loader import YamlLoader
from flowsint_core.templates.types import Template
+TEST_DIR = Path(__file__).parent
+
def test_yaml_loader():
- file = YamlLoader.get_template_from_file(
- "/Users/eliottmorcillo/Documents/Perso/projets/reconurge/flowsint/flowsint-core/tests/templates/example.yaml"
- )
+ file = YamlLoader.get_template_from_file(str(TEST_DIR / "example.yaml"))
assert isinstance(file, Template)
assert file.name == "shodan_ip_lookup"
assert file.category == "Ip"
@@ -21,3 +24,21 @@ def test_render_template():
url = YamlLoader.render_template(url_template, {"address": "8.8.8.8"})
assert url == "http://ip-api.com/json/8.8.8.8"
+
+
+def test_render_template_2():
+ url_template = "http://ip-api.com/json?ip={{address}}&domain={{domain}}"
+
+ url = YamlLoader.render_template(
+ url_template, {"address": "8.8.8.8", "domain": "mydomain.com"}
+ )
+
+ assert url == "http://ip-api.com/json?ip=8.8.8.8&domain=mydomain.com"
+ print(url)
+
+
+def test_invalid_method():
+ with pytest.raises(ValueError) as exc_info:
+ YamlLoader.get_template_from_file(str(TEST_DIR / "example-invalid.yaml"))
+
+ assert "not present in" in str(exc_info.value).lower()