A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(core): update dependencies + orchestrator
dextmorgn901ed15567c5
diff --git a/flowsint-core/src/flowsint_core/core/orchestrator.py b/flowsint-core/src/flowsint_core/core/orchestrator.pyindex d9d470a4..1b9d825a 100644--- a/flowsint-core/src/flowsint_core/core/orchestrator.py+++ b/flowsint-core/src/flowsint_core/core/orchestrator.py@@ -3,7 +3,7 @@ from datetime import datetimeimport timefrom pydantic import ValidationErrorfrom .enricher_base import Enricher-from .registry import EnricherRegistry+from flowsint_enrichers import ENRICHER_REGISTRYfrom .types import FlowBranch, FlowStepfrom .logger import Loggerfrom ..utils import to_json_serializable@@ -232,14 +232,14 @@ class FlowOrchestrator(Enricher):# Extract enricher name from nodeId (assuming format like "enricher_name-1234567890")enricher_name = node_id.split("-")[0]- if not EnricherRegistry.enricher_exists(enricher_name):+ if not ENRICHER_REGISTRY.enricher_exists(enricher_name):raise ValueError(f"Enricher '{enricher_name}' not found in registry")# Pass the step params to the enricher instanceenricher_params = (node.params if hasattr(node, "params") and node.params else {})- enricher = EnricherRegistry.get_enricher(+ enricher = ENRICHER_REGISTRY.get_enricher(enricher_name,self.sketch_id,self.scan_id,diff --git a/flowsint-core/src/flowsint_core/core/registry.py b/flowsint-core/src/flowsint_core/core/registry.pydeleted file mode 100644index cff123e0..00000000--- a/flowsint-core/src/flowsint_core/core/registry.py+++ /dev/null@@ -1,207 +0,0 @@-import inspect-from typing import Dict, Optional, Type, List, Any-from flowsint_core.core.enricher_base import Enricher--# Domain-related enrichers-from flowsint_enrichers.domain.to_subdomains import SubdomainEnricher-from flowsint_enrichers.domain.to_whois import WhoisEnricher-from flowsint_enrichers.domain.to_ip import ResolveEnricher-from flowsint_enrichers.domain.to_website import DomainToWebsiteEnricher-from flowsint_enrichers.domain.to_root_domain import DomainToRootDomain-from flowsint_enrichers.domain.to_asn import DomainToAsnEnricher-from flowsint_enrichers.domain.to_history import DomainToHistoryEnricher--# IP-related enrichers-from flowsint_enrichers.email.to_domains import EmailToDomainsEnricher-from flowsint_enrichers.individual.to_domains import IndividualToDomainsEnricher-from flowsint_enrichers.ip.to_domain import ReverseResolveEnricher-from flowsint_enrichers.ip.to_infos import IpToInfosEnricher-from flowsint_enrichers.ip.to_asn import IpToAsnEnricher-from flowsint_enrichers.ip.to_ports import IpToPortsEnricher--# ASN-related enrichers-from flowsint_enrichers.asn.to_cidrs import AsnToCidrsEnricher--# CIDR-related enrichers-from flowsint_enrichers.cidr.to_ips import CidrToIpsEnricher--# Social media enrichers-from flowsint_enrichers.organization.to_domains import OrgToDomainsEnricher-from flowsint_enrichers.social.to_maigret import MaigretEnricher--# Organization-related enrichers-from flowsint_enrichers.organization.to_asn import OrgToAsnEnricher-from flowsint_enrichers.organization.to_infos import OrgToInfosEnricher--# Cryptocurrency enrichers-from flowsint_enrichers.crypto.to_transactions import (- CryptoWalletAddressToTransactions,-)-from flowsint_enrichers.crypto.to_nfts import CryptoWalletAddressToNFTs--# Website-related enrichers-from flowsint_enrichers.website.to_crawler import WebsiteToCrawler-from flowsint_enrichers.website.to_links import WebsiteToLinks-from flowsint_enrichers.website.to_domain import WebsiteToDomainEnricher-from flowsint_enrichers.website.to_text import WebsiteToText-from flowsint_enrichers.website.to_webtrackers import WebsiteToWebtrackersEnricher--# Email-related enrichers-from flowsint_enrichers.email.to_gravatar import EmailToGravatarEnricher-from flowsint_enrichers.email.to_leaks import EmailToBreachesEnricher--# Phone-related enrichers--# Individual-related enrichers-from flowsint_enrichers.individual.to_org import IndividualToOrgEnricher--# Integration enrichers-from flowsint_enrichers.n8n.connector import N8nConnector--# Dummy-from flowsint_enrichers.ip.to_dummy_domains import IpToDummyDomainsEnricher-from flowsint_enrichers.domain.to_dummy_ip import DomainToDummyIpEnricher---class EnricherRegistry:-- _enrichers: Dict[str, Type[Enricher]] = {}-- @classmethod- def register(cls, enricher_class: Type[Enricher]) -> None:- cls._enrichers[enricher_class.name()] = enricher_class-- @classmethod- def enricher_exists(cls, name: str) -> bool:- return name in cls._enrichers-- @classmethod- def get_enricher(- cls, name: str, sketch_id: str, scan_id: str, **kwargs- ) -> Enricher:- if name not in cls._enrichers:- raise Exception(f"Enricher '{name}' not found")- return cls._enrichers[name](sketch_id=sketch_id, scan_id=scan_id, **kwargs)-- @classmethod- def _create_enricher_metadata(cls, enricher: Type[Enricher]) -> Dict[str, str]:- """Helper method to create enricher metadata dictionary."""- return {- "class_name": enricher.__name__,- "name": enricher.name(),- "module": enricher.__module__,- "description": enricher.__doc__,- "documentation": inspect.cleandoc(enricher.documentation()),- "category": enricher.category(),- "inputs": enricher.input_schema(),- "outputs": enricher.output_schema(),- "params": {},- "params_schema": enricher.get_params_schema(),- "required_params": enricher.required_params(),- "icon": enricher.icon(),- }-- @classmethod- def list(- cls, exclude: Optional[List[str]] = None, wobbly_type: Optional[bool] = False- ) -> List[Dict[str, Any]]:- if exclude is None:- exclude = []- return [- {- **cls._create_enricher_metadata(enricher),- "wobblyType": wobbly_type,- }- for enricher in cls._enrichers.values()- if enricher.name() not in exclude- ]-- @classmethod- def list_by_categories(cls) -> Dict[str, List[Dict[str, str]]]:- enrichers_by_category = {}- for _, enricher in cls._enrichers.items():- category = enricher.category()- if category not in enrichers_by_category:- enrichers_by_category[category] = []- enrichers_by_category[category].append(- cls._create_enricher_metadata(enricher)- )- return enrichers_by_category-- @classmethod- def list_by_input_type(- cls, input_type: str, exclude: Optional[List[str]] = []- ) -> List[Dict[str, str]]:- input_type_lower = input_type.lower()-- if input_type_lower == "any":- return [- cls._create_enricher_metadata(enricher)- for enricher in cls._enrichers.values()- if enricher.name() not in exclude- ]-- return [- cls._create_enricher_metadata(enricher)- for enricher in cls._enrichers.values()- if enricher.input_schema()["type"].lower() in ["any", input_type_lower]- and enricher.name() not in exclude- ]---# Register all enrichers--# Domain-related enrichers-EnricherRegistry.register(ReverseResolveEnricher)-EnricherRegistry.register(ResolveEnricher)-EnricherRegistry.register(SubdomainEnricher)-EnricherRegistry.register(WhoisEnricher)-EnricherRegistry.register(DomainToWebsiteEnricher)-EnricherRegistry.register(DomainToRootDomain)-EnricherRegistry.register(DomainToAsnEnricher)-EnricherRegistry.register(DomainToHistoryEnricher)--# IP-related enrichers-EnricherRegistry.register(IpToInfosEnricher)-EnricherRegistry.register(IpToAsnEnricher)-EnricherRegistry.register(IpToPortsEnricher)--# ASN-related enrichers-EnricherRegistry.register(AsnToCidrsEnricher)--# CIDR-related enrichers-EnricherRegistry.register(CidrToIpsEnricher)--# Social media enrichers-EnricherRegistry.register(MaigretEnricher)--# Organization-related enrichers-EnricherRegistry.register(OrgToAsnEnricher)-EnricherRegistry.register(OrgToInfosEnricher)-EnricherRegistry.register(OrgToDomainsEnricher)-# Cryptocurrency enrichers-EnricherRegistry.register(CryptoWalletAddressToTransactions)-EnricherRegistry.register(CryptoWalletAddressToNFTs)--# Website-related enrichers-EnricherRegistry.register(WebsiteToCrawler)-EnricherRegistry.register(WebsiteToLinks)-EnricherRegistry.register(WebsiteToDomainEnricher)-EnricherRegistry.register(WebsiteToWebtrackersEnricher)-EnricherRegistry.register(WebsiteToText)--# Email-related enrichers-EnricherRegistry.register(EmailToGravatarEnricher)-EnricherRegistry.register(EmailToBreachesEnricher)-EnricherRegistry.register(EmailToDomainsEnricher)--# Individual-related enrichers-EnricherRegistry.register(IndividualToOrgEnricher)-EnricherRegistry.register(IndividualToDomainsEnricher)--# Integration enrichers-EnricherRegistry.register(N8nConnector)--# Dummy-EnricherRegistry.register(IpToDummyDomainsEnricher)-EnricherRegistry.register(DomainToDummyIpEnricher)diff --git a/flowsint-core/src/flowsint_core/tasks/enricher.py b/flowsint-core/src/flowsint_core/tasks/enricher.pyindex fc009b30..81f0144f 100644--- a/flowsint-core/src/flowsint_core/tasks/enricher.py+++ b/flowsint-core/src/flowsint_core/tasks/enricher.py@@ -4,7 +4,7 @@ import asynciofrom dotenv import load_dotenvfrom typing import List, Optionalfrom celery import states-from flowsint_core.core.registry import EnricherRegistry+from flowsint_enrichers import ENRICHER_REGISTRY, load_all_enrichersfrom ..core.celery import celeryfrom ..core.postgre_db import SessionLocal, get_dbfrom ..core.graph_db import Neo4jConnection@@ -17,6 +17,9 @@ from flowsint_core.utils import to_json_serializableload_dotenv()+# Auto-discover and register all enrichers+load_all_enrichers()+URI = os.getenv("NEO4J_URI_BOLT")USERNAME = os.getenv("NEO4J_USERNAME")PASSWORD = os.getenv("NEO4J_PASSWORD")@@ -57,10 +60,10 @@ def run_enricher(sketch_id, {"message": f"Failed to create vault: {str(e)}"})- if not EnricherRegistry.enricher_exists(enricher_name):+ if not ENRICHER_REGISTRY.enricher_exists(enricher_name):raise ValueError(f"Enricher '{enricher_name}' not found in registry")- enricher = EnricherRegistry.get_enricher(+ enricher = ENRICHER_REGISTRY.get_enricher(name=enricher_name,sketch_id=sketch_id,scan_id=scan_id,diff --git a/flowsint-core/tests/import/test_entity_detection.py b/flowsint-core/tests/import/test_entity_detection.pyindex e0951c06..533e93e4 100644--- a/flowsint-core/tests/import/test_entity_detection.py+++ b/flowsint-core/tests/import/test_entity_detection.py@@ -44,7 +44,6 @@ def test_detection_mixed():results = []for obj in raw_obj:DetectedType = detect_type(obj)- print(DetectedType)if DetectedType:py_obj = DetectedType.from_string(obj)results.append(py_obj)