A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): expose params_schema for template enrichers in listing
dextmorgn725d321cda9c
diff --git a/flowsint-api/tests/test_enrichers_listing.py b/flowsint-api/tests/test_enrichers_listing.pynew file mode 100644index 00000000..74355061--- /dev/null+++ b/flowsint-api/tests/test_enrichers_listing.py@@ -0,0 +1,66 @@+"""Contract for GET /api/enrichers: template rows advertise their params."""++import uuid++from flowsint_core.core.auth import create_access_token+from flowsint_core.core.models import EnricherTemplate, Profile+++def _seed_template_owner(db_session, secrets):+ user = Profile(id=uuid.uuid4(), email="[email protected]", hashed_password="x")+ db_session.add(user)+ db_session.add(+ EnricherTemplate(+ id=uuid.uuid4(),+ name="github-lookup",+ description="Look a user up on GitHub",+ category="Username",+ version=1.0,+ content={+ "name": "github-lookup",+ "category": "Username",+ "version": 1.0,+ "input": {"type": "Username"},+ "request": {"url": "https://api.github.com/users/{{username}}"},+ "response": {},+ "output": {"type": "Username"},+ "secrets": secrets,+ },+ is_public=False,+ owner_id=user.id,+ )+ )+ db_session.commit()+ return {"Authorization": f"Bearer {create_access_token({'sub': user.email})}"}+++def test_template_enrichers_expose_params_schema(client, db_session):+ headers = _seed_template_owner(+ db_session,+ [{"name": "GITHUB_TOKEN", "required": True, "description": "A PAT"}],+ )++ res = client.get("/api/enrichers", headers=headers)++ assert res.status_code == 200+ [template] = [e for e in res.json() if e["name"] == "github-lookup"]+ assert template["params_schema"] == [+ {+ "name": "GITHUB_TOKEN",+ "type": "vaultSecret",+ "required": True,+ "description": "A PAT",+ }+ ]+ assert template["required_params"] is True+++def test_template_enricher_without_secrets_reports_no_params(client, db_session):+ headers = _seed_template_owner(db_session, [])++ res = client.get("/api/enrichers", headers=headers)++ assert res.status_code == 200+ [template] = [e for e in res.json() if e["name"] == "github-lookup"]+ assert template["params_schema"] == []+ assert template["required_params"] is Falsediff --git a/flowsint-core/src/flowsint_core/core/services/enricher_service.py b/flowsint-core/src/flowsint_core/core/services/enricher_service.pyindex 494e55ad..a51f249e 100644--- a/flowsint-core/src/flowsint_core/core/services/enricher_service.py+++ b/flowsint-core/src/flowsint_core/core/services/enricher_service.py@@ -5,12 +5,33 @@ Enricher service for managing enricher operations.from typing import Any, Dict, List, Optionalfrom uuid import UUID+from pydantic import ValidationErrorfrom sqlalchemy.orm import Session+from ...templates.types import TemplateSecret, params_schema_from_secretsfrom ..repositories import CustomTypeRepository, EnricherTemplateRepositoryfrom .base import BaseService+def template_params_schema(content: Any) -> List[Dict[str, Any]]:+ """Derive a params schema from a stored template's `secrets` block.++ Stored content is unvalidated JSON, so a secret that does not parse is+ dropped instead of failing the whole enricher listing.+ """+ if not isinstance(content, dict):+ return []++ secrets = []+ for raw_secret in content.get("secrets") or []:+ try:+ secrets.append(TemplateSecret.model_validate(raw_secret))+ except ValidationError:+ continue++ return params_schema_from_secrets(secrets)++class EnricherService(BaseService):"""Service for enricher operations and listing.@@ -50,6 +71,15 @@ class EnricherService(BaseService):template_enrichers = self._enricher_template_repo.get_by_owner(user_id, category)+ for template in template_enrichers:+ # Unmapped attributes: they reach the response because FastAPI's+ # encoder reads the instance __dict__, and SQLAlchemy never tries+ # to persist a key it has no column for.+ params_schema = template_params_schema(template.content)+ template.params_schema = params_schema # type: ignore[attr-defined]+ template.required_params = any( # type: ignore[attr-defined]+ param["required"] for param in params_schema+ )return [*base_enrichers, *template_enrichers]diff --git a/flowsint-core/src/flowsint_core/core/template_enricher.py b/flowsint-core/src/flowsint_core/core/template_enricher.pyindex f06f0dbe..3325a50a 100644--- a/flowsint-core/src/flowsint_core/core/template_enricher.py+++ b/flowsint-core/src/flowsint_core/core/template_enricher.py@@ -58,7 +58,11 @@ from flowsint_core.templates.loader.yaml_loader import (YamlLoader,validate_url_safe,)-from flowsint_core.templates.types import Template, TemplateRetryConfig+from flowsint_core.templates.types import (+ Template,+ TemplateRetryConfig,+ params_schema_from_secrets,+)from flowsint_types import FlowsintType, get_type@@ -92,15 +96,12 @@ class TemplateEnricher(Enricher):vault: Optional[VaultProtocol] = None,params: Optional[Dict[str, Any]] = None,) -> None:- # Build params schema from template secrets- params_schema = self._build_params_schema_from_template(template)-super().__init__(sketch_id=sketch_id,scan_id=scan_id,vault=vault,params=params,- params_schema=params_schema,+ params_schema=params_schema_from_secrets(template.secrets),)self.template = templateself.InputType = self._detect_type(self.template.input.type)@@ -109,21 +110,6 @@ class TemplateEnricher(Enricher):self._resolved_secrets: Dict[str, str] = {}self.raw_response: Dict[str, Any] | None = None- @staticmethod- def _build_params_schema_from_template(template: Template) -> List[Dict[str, Any]]:- """Convert template secrets to enricher params schema format."""- schema = []- for secret in template.secrets:- schema.append(- {- "name": secret.name,- "type": "vaultSecret",- "required": secret.required,- "description": secret.description or f"Secret: {secret.name}",- }- )- return schema-def _detect_type(self, input_type: str) -> type[FlowsintType]:"""Resolve a type name to its FlowsintType class."""# flowsint_types isn't py.typed, so get_type() resolves to Any herediff --git a/flowsint-core/src/flowsint_core/templates/types.py b/flowsint-core/src/flowsint_core/templates/types.pyindex 985bf5ab..ff2ff215 100644--- a/flowsint-core/src/flowsint_core/templates/types.py+++ b/flowsint-core/src/flowsint_core/templates/types.py@@ -1,4 +1,4 @@-from typing import List, Literal, Optional+from typing import Any, Dict, Iterable, List, Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Field@@ -79,6 +79,26 @@ class TemplateSecret(BaseModel):)+def params_schema_from_secrets(+ secrets: Iterable[TemplateSecret],+) -> List[Dict[str, Any]]:+ """Map a template's vault secrets to the enricher params_schema shape.++ Single definition on purpose: the running TemplateEnricher validates+ launch params against this, and the enricher listing advertises it to the+ UI. If the two drifted, the UI would collect params the enricher rejects.+ """+ return [+ {+ "name": secret.name,+ "type": "vaultSecret",+ "required": secret.required,+ "description": secret.description,+ }+ for secret in secrets+ ]++class TemplateHttpRequest(BaseModel):model_config = ConfigDict(extra="forbid", frozen=True)diff --git a/flowsint-core/tests/services/test_enricher_service.py b/flowsint-core/tests/services/test_enricher_service.pynew file mode 100644index 00000000..401574ed--- /dev/null+++ b/flowsint-core/tests/services/test_enricher_service.py@@ -0,0 +1,122 @@+"""Tests for EnricherService listing, focused on template-enricher params."""++from unittest.mock import MagicMock++from flowsint_core.core.repositories import (+ CustomTypeRepository,+ EnricherTemplateRepository,+)+from flowsint_core.core.services.enricher_service import EnricherService+from tests.factories import EnricherTemplateFactory, ProfileFactory+++def _content(secrets):+ return {+ "name": "example",+ "category": "Ip",+ "version": 1.0,+ "input": {"type": "Ip"},+ "request": {"url": "https://api.example.com/{{address}}"},+ "response": {},+ "output": {"type": "Ip"},+ "secrets": secrets,+ }+++class TestGetAllEnrichers:+ def _setup(self, db_session):+ ProfileFactory._meta.sqlalchemy_session = db_session+ EnricherTemplateFactory._meta.sqlalchemy_session = db_session++ def _make_service(self, db_session):+ return EnricherService(+ db=db_session,+ custom_type_repo=CustomTypeRepository(db_session),+ enricher_template_repo=EnricherTemplateRepository(db_session),+ )++ def _list(self, db_session, user):+ registry = MagicMock()+ registry.list.return_value = []+ return self._make_service(db_session).get_all_enrichers(None, user.id, registry)++ def test_template_secrets_become_params_schema(self, db_session):+ self._setup(db_session)+ user = ProfileFactory()+ EnricherTemplateFactory(+ owner=user,+ name="github-lookup",+ content=_content(+ [+ {+ "name": "GITHUB_TOKEN",+ "required": True,+ "description": "Personal access token",+ },+ {"name": "OPTIONAL_KEY", "required": False},+ ]+ ),+ )++ [template] = self._list(db_session, user)++ assert template.params_schema == [+ {+ "name": "GITHUB_TOKEN",+ "type": "vaultSecret",+ "required": True,+ "description": "Personal access token",+ },+ {+ "name": "OPTIONAL_KEY",+ "type": "vaultSecret",+ "required": False,+ "description": None,+ },+ ]+ assert template.required_params is True++ def test_template_without_required_secrets_is_not_required(self, db_session):+ self._setup(db_session)+ user = ProfileFactory()+ EnricherTemplateFactory(+ owner=user,+ content=_content([{"name": "OPTIONAL_KEY", "required": False}]),+ )++ [template] = self._list(db_session, user)++ assert template.required_params is False++ def test_template_without_secrets_gets_an_empty_schema(self, db_session):+ self._setup(db_session)+ user = ProfileFactory()+ EnricherTemplateFactory(owner=user, content=_content([]))++ [template] = self._list(db_session, user)++ assert template.params_schema == []+ assert template.required_params is False++ def test_malformed_secret_is_dropped_rather_than_failing_the_listing(+ self, db_session+ ):+ self._setup(db_session)+ user = ProfileFactory()+ EnricherTemplateFactory(+ owner=user,+ content=_content([{"nope": 1}, {"name": "GOOD_KEY", "required": True}]),+ )++ [template] = self._list(db_session, user)++ assert [param["name"] for param in template.params_schema] == ["GOOD_KEY"]++ def test_template_with_non_dict_content_gets_an_empty_schema(self, db_session):+ self._setup(db_session)+ user = ProfileFactory()+ EnricherTemplateFactory(owner=user, content=["not", "a", "dict"])++ [template] = self._list(db_session, user)++ assert template.params_schema == []