Skip to content

A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.

feat: parse httpUrl on db insert

dextmorgn306b2958ba9d
diff --git a/flowsint-core/src/flowsint_core/core/logger.py b/flowsint-core/src/flowsint_core/core/logger.py
index eb6905b1..a1df0fab 100644
--- a/flowsint-core/src/flowsint_core/core/logger.py
+++ b/flowsint-core/src/flowsint_core/core/logger.py
@@ -1,90 +1,273 @@
+"""
+Modern Logger implementation with Singleton pattern and batch insertion.
+
+Architecture:
+- Singleton: Thread-safe instance
+- Immediate event emission: Real-time display in UI
+- Batched database insertion: Performance optimization
+- SOLID principles: Dependency injection via protocols
+"""
+from typing import Dict, Union, Optional
from uuid import UUID
-from typing import Any, Dict, Union
+import threading
+import time
+import atexit
+from queue import Queue
+from .enums import EventLevel
from .models import Log
-from ..tasks.event import emit_event_task
from .postgre_db import get_db
-from .enums import EventLevel
-import logging
+from ..tasks.event import emit_event_task
+
+
+class LoggerSingleton:
+ """
+ Thread-safe Singleton Logger with batched database insertion.
+
+ Features:
+ - Immediate event emission for real-time UI updates
+ - Batched database writes for performance
+ - Thread-safe operations
+ - Automatic flush on shutdown
+ """
+
+ _instance: Optional['LoggerSingleton'] = None
+ _lock = threading.Lock()
+
+ def __new__(
+ cls,
+ batch_size: int = 50,
+ flush_interval: float = 2.0,
+ auto_start: bool = True
+ ):
+ """Thread-safe singleton implementation using double-checked locking."""
+ if cls._instance is None:
+ with cls._lock:
+ # Double-check to prevent race condition
+ if cls._instance is None:
+ cls._instance = super().__new__(cls)
+ cls._instance._initialized = False
+ return cls._instance
+
+ def __init__(
+ self,
+ batch_size: int = 50,
+ flush_interval: float = 2.0,
+ auto_start: bool = True
+ ):
+ """
+ Initialize the Logger singleton.
+
+ Args:
+ batch_size: Number of logs to batch before writing to DB
+ flush_interval: Time in seconds between automatic flushes
+ auto_start: Whether to start the batch worker automatically
+ """
+ # Only initialize once
+ if self._initialized:
+ return
+
+ self._initialized = True
+ self._batch_size = batch_size
+ self._flush_interval = flush_interval
-logger = logging.getLogger(__name__)
+ # Queue for batch insertion
+ self._log_queue: Queue = Queue()
+ # Worker thread for batch processing
+ self._worker_thread: Optional[threading.Thread] = None
+ self._shutdown_event = threading.Event()
-class Logger:
- @staticmethod
- def _create_log(
- sketch_id: Union[str, UUID], level: EventLevel, content: str
- ) -> Log:
- """Create a log entry in the database"""
- db = next(get_db())
+ # Register cleanup on exit
+ atexit.register(self.shutdown)
+
+ if auto_start:
+ self.start()
+
+ def start(self) -> None:
+ """Start the batch worker thread."""
+ if self._worker_thread is None or not self._worker_thread.is_alive():
+ self._shutdown_event.clear()
+ self._worker_thread = threading.Thread(
+ target=self._batch_worker,
+ daemon=True,
+ name="LoggerBatchWorker"
+ )
+ self._worker_thread.start()
+
+ def shutdown(self) -> None:
+ """Shutdown the logger and flush all pending logs."""
+ self._shutdown_event.set()
+ if self._worker_thread and self._worker_thread.is_alive():
+ self._worker_thread.join(timeout=5.0)
+ # Final flush
+ self._flush_batch(force=True)
+
+ def _batch_worker(self) -> None:
+ """Background worker that flushes logs periodically."""
+ while not self._shutdown_event.is_set():
+ time.sleep(self._flush_interval)
+ self._flush_batch()
+
+ def _flush_batch(self, force: bool = False) -> None:
+ """
+ Flush pending logs to database.
+
+ Args:
+ force: If True, flush all logs regardless of batch size
+ """
+ if self._log_queue.empty():
+ return
+
+ logs_to_insert = []
+
+ # Collect logs from queue
+ while not self._log_queue.empty() and (force or len(logs_to_insert) < self._batch_size):
+ try:
+ logs_to_insert.append(self._log_queue.get_nowait())
+ except:
+ break
+
+ if not logs_to_insert:
+ return
+
+ # Batch insert to database
try:
- log = Log(sketch_id=str(sketch_id), type=level.value, content=content)
- db.add(log)
+ db = next(get_db())
+ except (StopIteration, RuntimeError) as e:
+ # Generator exhausted or error getting DB - log to standard logging
+ import logging
+ logging.error(f"Failed to get database session: {e}")
+ return
+
+ try:
+ log_objects = []
+ for sketch_id, level, content in logs_to_insert:
+ log = Log(
+ sketch_id=str(sketch_id),
+ type=level.value,
+ content=content
+ )
+ log_objects.append(log)
+
+ db.add_all(log_objects)
db.commit()
- db.refresh(log)
- return log
+
+ # Refresh to get IDs (needed for event emission tracking)
+ for log in log_objects:
+ db.refresh(log)
+
+ except Exception as e:
+ db.rollback()
+ # Log to standard logging as fallback
+ import logging
+ logging.error(f"Failed to batch insert logs: {e}")
finally:
db.close()
- @staticmethod
- def info(sketch_id: Union[str, UUID], message: Dict):
- """Log an info message"""
- log = Logger._create_log(sketch_id, EventLevel.INFO, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.INFO, message]
- )
-
- @staticmethod
- def error(sketch_id: Union[str, UUID], message: Dict):
- """Log an error message"""
- log = Logger._create_log(sketch_id, EventLevel.FAILED, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.FAILED, message]
- )
-
- @staticmethod
- def warn(sketch_id: Union[str, UUID], message: Dict):
- """Log a warning message"""
- log = Logger._create_log(sketch_id, EventLevel.WARNING, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.WARNING, message]
- )
-
- @staticmethod
- def debug(sketch_id: Union[str, UUID], message: Dict):
- """Log a debug message"""
- log = Logger._create_log(sketch_id, EventLevel.DEBUG, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.DEBUG, message]
- )
-
- @staticmethod
- def graph_append(sketch_id: Union[str, UUID], message: Dict):
- """Log an insert event to the graph"""
- log = Logger._create_log(sketch_id, EventLevel.GRAPH_APPEND, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.GRAPH_APPEND, message]
- )
-
- @staticmethod
- def success(sketch_id: Union[str, UUID], message: Dict):
- """Log a success message"""
- log = Logger._create_log(sketch_id, EventLevel.SUCCESS, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.SUCCESS, message]
- )
-
- @staticmethod
- def completed(sketch_id: Union[str, UUID], message: Dict):
- """Log a completed message"""
- log = Logger._create_log(sketch_id, EventLevel.COMPLETED, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.COMPLETED, message]
- )
-
- @staticmethod
- def pending(sketch_id: Union[str, UUID], message: Dict):
- """Log a pending message"""
- log = Logger._create_log(sketch_id, EventLevel.PENDING, message)
- emit_event_task.apply(
- args=[str(log.id), str(sketch_id), EventLevel.PENDING, message]
- )
+ def _emit_event(
+ self,
+ log_id: str,
+ sketch_id: str,
+ level: EventLevel,
+ content: Dict
+ ) -> None:
+ """
+ Emit event immediately for real-time display.
+
+ Args:
+ log_id: Log entry ID
+ sketch_id: Sketch ID for event routing
+ level: Log level
+ content: Log content
+ """
+ try:
+ emit_event_task.apply(
+ args=[log_id, str(sketch_id), level, content]
+ )
+ except Exception as e:
+ # Don't let event emission errors break logging
+ import logging
+ logging.error(f"Failed to emit event: {e}")
+
+ def _log(
+ self,
+ sketch_id: Union[str, UUID],
+ level: EventLevel,
+ content: Dict
+ ) -> None:
+ """
+ Internal logging method.
+
+ Process:
+ 1. Emit event immediately (real-time UI)
+ 2. Queue for batch insertion (performance)
+
+ Args:
+ sketch_id: Sketch ID for log routing
+ level: Log level
+ content: Log content/message
+ """
+ # Generate a temporary ID for immediate event emission
+ import uuid
+ temp_log_id = str(uuid.uuid4())
+
+ # 1. IMMEDIATE: Emit event for real-time display
+ self._emit_event(temp_log_id, str(sketch_id), level, content)
+
+ # 2. BATCHED: Queue for database insertion
+ self._log_queue.put((str(sketch_id), level, content))
+
+ # 3. Check if we should flush immediately (batch size reached)
+ if self._log_queue.qsize() >= self._batch_size:
+ self._flush_batch()
+
+ # Public API methods
+
+ def info(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log an info message."""
+ self._log(sketch_id, EventLevel.INFO, message)
+
+ def error(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log an error message."""
+ self._log(sketch_id, EventLevel.FAILED, message)
+
+ def warn(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log a warning message."""
+ self._log(sketch_id, EventLevel.WARNING, message)
+
+ def debug(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log a debug message."""
+ self._log(sketch_id, EventLevel.DEBUG, message)
+
+ def success(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log a success message."""
+ self._log(sketch_id, EventLevel.SUCCESS, message)
+
+ def completed(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log a completed message."""
+ self._log(sketch_id, EventLevel.COMPLETED, message)
+
+ def pending(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log a pending message."""
+ self._log(sketch_id, EventLevel.PENDING, message)
+
+ def graph_append(self, sketch_id: Union[str, UUID], message: Dict) -> None:
+ """Log a graph append operation."""
+ self._log(sketch_id, EventLevel.GRAPH_APPEND, message)
+
+ def flush(self) -> None:
+ """Manually flush all pending logs to database."""
+ self._flush_batch(force=True)
+
+ @property
+ def queue_size(self) -> int:
+ """Get current size of the log queue."""
+ return self._log_queue.qsize()
+
+
+# Global singleton instance
+_logger = LoggerSingleton()
+
+
+# Export the singleton instance as Logger
+Logger = _logger
diff --git a/flowsint-core/src/flowsint_core/core/logger_old.py b/flowsint-core/src/flowsint_core/core/logger_old.py
new file mode 100644
index 00000000..eb6905b1
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/core/logger_old.py
@@ -0,0 +1,90 @@
+from uuid import UUID
+from typing import Any, Dict, Union
+from .models import Log
+from ..tasks.event import emit_event_task
+from .postgre_db import get_db
+from .enums import EventLevel
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Logger:
+ @staticmethod
+ def _create_log(
+ sketch_id: Union[str, UUID], level: EventLevel, content: str
+ ) -> Log:
+ """Create a log entry in the database"""
+ db = next(get_db())
+ try:
+ log = Log(sketch_id=str(sketch_id), type=level.value, content=content)
+ db.add(log)
+ db.commit()
+ db.refresh(log)
+ return log
+ finally:
+ db.close()
+
+ @staticmethod
+ def info(sketch_id: Union[str, UUID], message: Dict):
+ """Log an info message"""
+ log = Logger._create_log(sketch_id, EventLevel.INFO, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.INFO, message]
+ )
+
+ @staticmethod
+ def error(sketch_id: Union[str, UUID], message: Dict):
+ """Log an error message"""
+ log = Logger._create_log(sketch_id, EventLevel.FAILED, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.FAILED, message]
+ )
+
+ @staticmethod
+ def warn(sketch_id: Union[str, UUID], message: Dict):
+ """Log a warning message"""
+ log = Logger._create_log(sketch_id, EventLevel.WARNING, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.WARNING, message]
+ )
+
+ @staticmethod
+ def debug(sketch_id: Union[str, UUID], message: Dict):
+ """Log a debug message"""
+ log = Logger._create_log(sketch_id, EventLevel.DEBUG, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.DEBUG, message]
+ )
+
+ @staticmethod
+ def graph_append(sketch_id: Union[str, UUID], message: Dict):
+ """Log an insert event to the graph"""
+ log = Logger._create_log(sketch_id, EventLevel.GRAPH_APPEND, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.GRAPH_APPEND, message]
+ )
+
+ @staticmethod
+ def success(sketch_id: Union[str, UUID], message: Dict):
+ """Log a success message"""
+ log = Logger._create_log(sketch_id, EventLevel.SUCCESS, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.SUCCESS, message]
+ )
+
+ @staticmethod
+ def completed(sketch_id: Union[str, UUID], message: Dict):
+ """Log a completed message"""
+ log = Logger._create_log(sketch_id, EventLevel.COMPLETED, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.COMPLETED, message]
+ )
+
+ @staticmethod
+ def pending(sketch_id: Union[str, UUID], message: Dict):
+ """Log a pending message"""
+ log = Logger._create_log(sketch_id, EventLevel.PENDING, message)
+ emit_event_task.apply(
+ args=[str(log.id), str(sketch_id), EventLevel.PENDING, message]
+ )
diff --git a/flowsint-core/src/flowsint_core/core/logger_protocols.py b/flowsint-core/src/flowsint_core/core/logger_protocols.py
new file mode 100644
index 00000000..f6d71c48
--- /dev/null
+++ b/flowsint-core/src/flowsint_core/core/logger_protocols.py
@@ -0,0 +1,52 @@
+"""
+Protocols and interfaces for the Logger system.
+Following SOLID principles with Dependency Inversion.
+"""
+from typing import Protocol, Any, Dict
+from uuid import UUID
+from .models import Log
+from .enums import EventLevel
+
+
+class DatabaseSession(Protocol):
+ """Protocol for database session operations."""
+
+ def add(self, instance: Any) -> None:
+ """Add an instance to the session."""
+ ...
+
+ def add_all(self, instances: list[Any]) -> None:
+ """Add multiple instances to the session."""
+ ...
+
+ def commit(self) -> None:
+ """Commit the current transaction."""
+ ...
+
+ def refresh(self, instance: Any) -> None:
+ """Refresh an instance from the database."""
+ ...
+
+ def close(self) -> None:
+ """Close the session."""
+ ...
+
+
+class EventEmitter(Protocol):
+ """Protocol for event emission (Redis pub/sub)."""
+
+ def emit(self, log_id: str, sketch_id: str, level: EventLevel, content: Dict) -> None:
+ """Emit a log event to Redis for real-time display."""
+ ...
+
+
+class LogStorage(Protocol):
+ """Protocol for log storage operations."""
+
+ def store_log(self, sketch_id: str, level: EventLevel, content: Dict) -> Log:
+ """Store a single log entry."""
+ ...
+
+ def store_logs_batch(self, logs_data: list[tuple[str, EventLevel, Dict]]) -> list[Log]:
+ """Store multiple log entries in a batch."""
+ ...
diff --git a/flowsint-core/src/flowsint_core/core/transform_base.py b/flowsint-core/src/flowsint_core/core/transform_base.py
index bcd32113..3233b8ab 100644
--- a/flowsint-core/src/flowsint_core/core/transform_base.py
+++ b/flowsint-core/src/flowsint_core/core/transform_base.py
@@ -7,8 +7,6 @@ from .logger import Logger
from .vault import VaultProtocol
from .graph_service import GraphService, create_graph_service
from ..utils import resolve_type
-import warnings
-
class InvalidTransformParams(Exception):
pass
@@ -498,10 +496,12 @@ class Transform(ABC):
key_value = self._extract_primary_value(key_value)
# Merge object properties with any overrides, but skip nested Pydantic objects
+ # Use model_dump(mode="json") to properly serialize Pydantic types (e.g., HttpUrl)
+ obj_dict = obj.model_dump(mode="json") if hasattr(obj, "model_dump") else obj.dict()
obj_properties = {}
- for k, v in obj.__dict__.items():
- # Skip nested Pydantic objects
- if not isinstance(v, BaseModel):
+ for k, v in obj_dict.items():
+ # Skip nested Pydantic objects (represented as dicts after model_dump)
+ if not isinstance(v, dict):
obj_properties[k] = v
obj_properties.update(properties)
properties = obj_properties
@@ -585,20 +585,32 @@ class Transform(ABC):
# Extract from_node info
from_node_type = from_obj.__class__.__name__.lower()
from_primary_field = self._get_primary_field(from_obj)
- from_key_value = getattr(from_obj, from_primary_field)
- # If key_value is a Pydantic model, extract its primary value
- if isinstance(from_key_value, BaseModel):
- from_key_value = self._extract_primary_value(from_key_value)
+ # Use model_dump to properly serialize Pydantic types (e.g., HttpUrl)
+ from_obj_dict = from_obj.model_dump(mode="json") if hasattr(from_obj, "model_dump") else from_obj.dict()
+ from_key_value = from_obj_dict.get(from_primary_field)
+
+ # If key_value is still a dict (nested Pydantic model), extract its primary value
+ if isinstance(from_key_value, dict):
+ # Get the raw nested object to extract its primary value
+ nested_obj = getattr(from_obj, from_primary_field)
+ if isinstance(nested_obj, BaseModel):
+ from_key_value = self._extract_primary_value(nested_obj)
# Extract to_node info
to_node_type = to_obj.__class__.__name__.lower()
to_primary_field = self._get_primary_field(to_obj)
- to_key_value = getattr(to_obj, to_primary_field)
- # If key_value is a Pydantic model, extract its primary value
- if isinstance(to_key_value, BaseModel):
- to_key_value = self._extract_primary_value(to_key_value)
+ # Use model_dump to properly serialize Pydantic types (e.g., HttpUrl)
+ to_obj_dict = to_obj.model_dump(mode="json") if hasattr(to_obj, "model_dump") else to_obj.dict()
+ to_key_value = to_obj_dict.get(to_primary_field)
+
+ # If key_value is still a dict (nested Pydantic model), extract its primary value
+ if isinstance(to_key_value, dict):
+ # Get the raw nested object to extract its primary value
+ nested_obj = getattr(to_obj, to_primary_field)
+ if isinstance(nested_obj, BaseModel):
+ to_key_value = self._extract_primary_value(nested_obj)
self._graph_service.create_relationship(
from_type=from_node_type,
@@ -648,13 +660,20 @@ class Transform(ABC):
"""
Extract the primitive value from a Pydantic object recursively.
If the primary field is itself a Pydantic object, recursively extract its primary value.
+ Uses model_dump to properly serialize Pydantic types like HttpUrl.
"""
primary_field = self._get_primary_field(obj)
- value = getattr(obj, primary_field)
- # If the value is still a Pydantic model, recursively extract
- if isinstance(value, BaseModel):
- return self._extract_primary_value(value)
+ # Use model_dump to properly serialize Pydantic types (e.g., HttpUrl)
+ obj_dict = obj.model_dump(mode="json") if hasattr(obj, "model_dump") else obj.dict()
+ value = obj_dict.get(primary_field)
+
+ # If the value is still a dict (nested Pydantic model), recursively extract
+ if isinstance(value, dict):
+ # Get the raw nested object to recursively extract
+ nested_obj = getattr(obj, primary_field)
+ if isinstance(nested_obj, BaseModel):
+ return self._extract_primary_value(nested_obj)
return value
diff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.py
index e55683c9..26fcd52e 100644
--- a/flowsint-core/src/flowsint_core/tasks/flow.py
+++ b/flowsint-core/src/flowsint_core/tasks/flow.py
@@ -23,7 +23,6 @@ PASSWORD = os.getenv("NEO4J_PASSWORD")
neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)
db: Session = next(get_db())
-logger = Logger()
@celery.task(name="run_flow", bind=True)
diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.py
index fe12b5a8..0c6e657b 100644
--- a/flowsint-core/src/flowsint_core/tasks/transform.py
+++ b/flowsint-core/src/flowsint_core/tasks/transform.py
@@ -23,7 +23,6 @@ PASSWORD = os.getenv("NEO4J_PASSWORD")
neo4j_connection = Neo4jConnection(URI, USERNAME, PASSWORD)
db: Session = next(get_db())
-logger = Logger()
@celery.task(name="run_transform", bind=True)
diff --git a/flowsint-core/tests/test_logger_singleton.py b/flowsint-core/tests/test_logger_singleton.py
new file mode 100644
index 00000000..29d7d92d
--- /dev/null
+++ b/flowsint-core/tests/test_logger_singleton.py
@@ -0,0 +1,425 @@
+"""
+Comprehensive tests for the LoggerSingleton implementation.
+
+Tests cover:
+- Singleton pattern behavior
+- Thread safety
+- Batch insertion
+- Event emission
+- Queue management
+- Shutdown behavior
+"""
+import pytest
+import threading
+import time
+from unittest.mock import Mock, patch, MagicMock
+from uuid import uuid4
+
+from flowsint_core.core.logger import LoggerSingleton
+from flowsint_core.core.enums import EventLevel
+
+
+def mock_db_session():
+ """Mock database session."""
+ session = Mock()
+ session.add = Mock()
+ session.add_all = Mock()
+ session.commit = Mock()
+ session.refresh = Mock()
+ session.rollback = Mock()
+ session.close = Mock()
+ return session
+
+
+def mock_get_db(mock_db_session):
+ """Mock the get_db generator."""
+ with patch('flowsint_core.core.logger.get_db') as mock:
+ mock.return_value = iter([mock_db_session])
+ yield mock
+
+
+def mock_emit_event():
+ """Mock the emit_event_task."""
+ with patch('flowsint_core.core.logger.emit_event_task') as mock:
+ mock.apply = Mock()
+ yield mock
+
+
+def logger_instance(mock_get_db, mock_emit_event):
+ """Create a fresh logger instance for testing."""
+ # Reset singleton
+ LoggerSingleton._instance = None
+ logger = LoggerSingleton(
+ batch_size=5,
+ flush_interval=0.5,
+ auto_start=False # Don't start worker for most tests
+ )
+ yield logger
+ # Cleanup
+ logger.shutdown()
+
+
+class TestSingletonPattern:
+ """Test the Singleton pattern implementation."""
+
+ def test_singleton_returns_same_instance(self, mock_get_db, mock_emit_event):
+ """Test that multiple calls return the same instance."""
+ LoggerSingleton._instance = None
+ logger1 = LoggerSingleton()
+ logger2 = LoggerSingleton()
+
+ assert logger1 is logger2
+ assert id(logger1) == id(logger2)
+
+ logger1.shutdown()
+
+ def test_singleton_is_thread_safe(self, mock_get_db, mock_emit_event):
+ """Test that singleton is thread-safe."""
+ LoggerSingleton._instance = None
+ instances = []
+
+ def create_logger():
+ logger = LoggerSingleton()
+ instances.append(logger)
+
+ threads = [threading.Thread(target=create_logger) for _ in range(10)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+
+ # All instances should be the same
+ first_instance = instances[0]
+ assert all(logger is first_instance for logger in instances)
+
+ first_instance.shutdown()
+
+
+class TestLoggingMethods:
+ """Test the various logging methods."""
+
+ def test_info_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test info logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test info"}
+
+ logger_instance.info(sketch_id, message)
+
+ # Event should be emitted immediately
+ mock_emit_event.apply.assert_called_once()
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[1] == sketch_id
+ assert call_args[2] == EventLevel.INFO
+ assert call_args[3] == message
+
+ # Log should be queued
+ assert logger_instance.queue_size == 1
+
+ def test_error_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test error logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test error"}
+
+ logger_instance.error(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.FAILED
+
+ def test_warn_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test warning logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test warning"}
+
+ logger_instance.warn(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.WARNING
+
+ def test_debug_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test debug logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test debug"}
+
+ logger_instance.debug(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.DEBUG
+
+ def test_success_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test success logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test success"}
+
+ logger_instance.success(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.SUCCESS
+
+ def test_completed_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test completed logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test completed"}
+
+ logger_instance.completed(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.COMPLETED
+
+ def test_pending_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test pending logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test pending"}
+
+ logger_instance.pending(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.PENDING
+
+ def test_graph_append_logs_correctly(self, logger_instance, mock_emit_event):
+ """Test graph append logging."""
+ sketch_id = str(uuid4())
+ message = {"message": "Test graph append"}
+
+ logger_instance.graph_append(sketch_id, message)
+
+ call_args = mock_emit_event.apply.call_args[1]['args']
+ assert call_args[2] == EventLevel.GRAPH_APPEND
+
+
+class TestBatchInsertion:
+ """Test batch insertion behavior."""
+
+ def test_logs_are_queued(self, logger_instance):
+ """Test that logs are queued before batch insertion."""
+ sketch_id = str(uuid4())
+
+ for i in range(3):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ assert logger_instance.queue_size == 3
+
+ def test_batch_flush_when_size_reached(self, logger_instance, mock_db_session):
+ """Test that batch flushes when batch_size is reached."""
+ sketch_id = str(uuid4())
+
+ # Logger has batch_size=5
+ for i in range(5):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ # Should trigger automatic flush
+ time.sleep(0.1) # Give it time to flush
+
+ # Verify database operations
+ mock_db_session.add_all.assert_called_once()
+ mock_db_session.commit.assert_called_once()
+
+ def test_manual_flush(self, logger_instance, mock_db_session):
+ """Test manual flush operation."""
+ sketch_id = str(uuid4())
+
+ # Add some logs (less than batch size)
+ for i in range(3):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ assert logger_instance.queue_size == 3
+
+ # Manual flush
+ logger_instance.flush()
+
+ # Queue should be empty
+ assert logger_instance.queue_size == 0
+
+ # Verify database operations
+ mock_db_session.add_all.assert_called_once()
+ mock_db_session.commit.assert_called_once()
+
+ def test_batch_worker_flushes_periodically(self, mock_get_db, mock_emit_event, mock_db_session):
+ """Test that batch worker flushes logs periodically."""
+ LoggerSingleton._instance = None
+ logger = LoggerSingleton(
+ batch_size=100, # High batch size
+ flush_interval=0.3, # Short interval for testing
+ auto_start=True # Start worker
+ )
+
+ sketch_id = str(uuid4())
+
+ # Add some logs
+ for i in range(3):
+ logger.info(sketch_id, {"message": f"Log {i}"})
+
+ # Wait for automatic flush (should happen within flush_interval)
+ time.sleep(0.5)
+
+ # Verify flush happened
+ assert mock_db_session.add_all.call_count >= 1
+
+ logger.shutdown()
+
+ def test_empty_queue_flush_does_nothing(self, logger_instance, mock_db_session):
+ """Test that flushing an empty queue does nothing."""
+ logger_instance.flush()
+
+ # No database operations should occur
+ mock_db_session.add_all.assert_not_called()
+ mock_db_session.commit.assert_not_called()
+
+
+class TestThreadSafety:
+ """Test thread safety of logging operations."""
+
+ def test_concurrent_logging(self, logger_instance, mock_emit_event):
+ """Test that concurrent logging from multiple threads works correctly."""
+ sketch_id = str(uuid4())
+ num_threads = 10
+ logs_per_thread = 10
+
+ def log_messages():
+ for i in range(logs_per_thread):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ threads = [threading.Thread(target=log_messages) for _ in range(num_threads)]
+
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+
+ # All logs should be queued or processed
+ expected_total = num_threads * logs_per_thread
+ assert mock_emit_event.apply.call_count == expected_total
+
+ def test_concurrent_flush(self, logger_instance, mock_db_session):
+ """Test that concurrent flush operations are safe."""
+ sketch_id = str(uuid4())
+
+ # Add some logs
+ for i in range(20):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ # Flush concurrently from multiple threads
+ def flush():
+ logger_instance.flush()
+
+ threads = [threading.Thread(target=flush) for _ in range(5)]
+
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+
+ # Queue should be empty
+ assert logger_instance.queue_size == 0
+
+ # At least one flush should have happened
+ assert mock_db_session.add_all.call_count >= 1
+
+
+class TestShutdown:
+ """Test shutdown behavior."""
+
+ def test_shutdown_flushes_pending_logs(self, mock_get_db, mock_emit_event, mock_db_session):
+ """Test that shutdown flushes all pending logs."""
+ LoggerSingleton._instance = None
+ logger = LoggerSingleton(batch_size=100, auto_start=False)
+
+ sketch_id = str(uuid4())
+
+ # Add logs
+ for i in range(10):
+ logger.info(sketch_id, {"message": f"Log {i}"})
+
+ assert logger.queue_size == 10
+
+ # Shutdown should flush
+ logger.shutdown()
+
+ # Queue should be empty
+ assert logger.queue_size == 0
+
+ # Database operations should have occurred
+ mock_db_session.add_all.assert_called_once()
+ mock_db_session.commit.assert_called_once()
+
+ def test_shutdown_stops_worker_thread(self, mock_get_db, mock_emit_event):
+ """Test that shutdown stops the worker thread."""
+ LoggerSingleton._instance = None
+ logger = LoggerSingleton(batch_size=10, flush_interval=0.5, auto_start=True)
+
+ assert logger._worker_thread.is_alive()
+
+ logger.shutdown()
+
+ # Give thread time to stop
+ time.sleep(0.1)
+
+ assert not logger._worker_thread.is_alive() or logger._shutdown_event.is_set()
+
+
+class TestErrorHandling:
+ """Test error handling in various scenarios."""
+
+ def test_database_error_does_not_crash(self, logger_instance, mock_db_session):
+ """Test that database errors don't crash the logger."""
+ mock_db_session.commit.side_effect = Exception("Database error")
+
+ sketch_id = str(uuid4())
+ logger_instance.info(sketch_id, {"message": "Test"})
+
+ # Flush should handle the error gracefully
+ logger_instance.flush()
+
+ # Logger should still be operational
+ logger_instance.info(sketch_id, {"message": "After error"})
+
+ def test_event_emission_error_does_not_crash(self, logger_instance, mock_emit_event):
+ """Test that event emission errors don't crash the logger."""
+ mock_emit_event.apply.side_effect = Exception("Event emission error")
+
+ sketch_id = str(uuid4())
+
+ # Should not raise exception
+ logger_instance.info(sketch_id, {"message": "Test"})
+
+ # Log should still be queued despite event emission failure
+ assert logger_instance.queue_size == 1
+
+
+class TestPerformance:
+ """Test performance characteristics."""
+
+ def test_high_volume_logging(self, logger_instance, mock_emit_event):
+ """Test that logger handles high volume of logs."""
+ sketch_id = str(uuid4())
+ num_logs = 1000
+
+ start_time = time.time()
+
+ for i in range(num_logs):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ elapsed = time.time() - start_time
+
+ # Should handle 1000 logs in reasonable time (< 1 second)
+ assert elapsed < 1.0
+
+ # All events should be emitted
+ assert mock_emit_event.apply.call_count == num_logs
+
+ def test_batch_insertion_is_efficient(self, logger_instance, mock_db_session):
+ """Test that batch insertion reduces database calls."""
+ sketch_id = str(uuid4())
+
+ # Log 50 messages (batch_size is 5)
+ for i in range(50):
+ logger_instance.info(sketch_id, {"message": f"Log {i}"})
+
+ logger_instance.flush()
+
+ # Should make fewer calls than number of logs (due to batching)
+ # With batch_size=5 and 50 logs, should make ~10 batch inserts
+ assert mock_db_session.add_all.call_count <= 15 # Some tolerance
diff --git a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
index 85c9ec91..70aae484 100644
--- a/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
+++ b/flowsint-transforms/src/flowsint_transforms/domain/to_subdomains.py
@@ -5,7 +5,6 @@ from flowsint_types.domain import Domain
from flowsint_core.utils import is_valid_domain
from flowsint_core.core.logger import Logger
from tools.network.subfinder import SubfinderTool
-from flowsint_core.core.logger import Logger
class SubdomainTransform(Transform):
diff --git a/flowsint-transforms/tests/transforms/domain/test_to_website.py b/flowsint-transforms/tests/transforms/domain/test_to_website.py
new file mode 100644
index 00000000..80d26474
--- /dev/null
+++ b/flowsint-transforms/tests/transforms/domain/test_to_website.py
@@ -0,0 +1,113 @@
+import pytest
+from flowsint_transforms.domain.to_website import DomainToWebsiteTransform
+from flowsint_types.domain import Domain
+from flowsint_types.website import Website
+from pydantic import HttpUrl
+
+
+def transform():
+ """Create transform instance for testing."""
+ return DomainToWebsiteTransform(sketch_id="test_sketch", scan_id="test_scan")
+
+
+def test_website_object_serialization(transform):
+ """Test that Website objects with HttpUrl can be properly serialized."""
+ # Create a website object with HttpUrl
+ domain = Domain(domain="example.com")
+ website = Website(
+ url=HttpUrl("https://example.com"),
+ domain=domain,
+ active=True,
+ redirects=[HttpUrl("https://www.example.com")]
+ )
+
+ # Verify the Website object is created correctly
+ assert isinstance(website.url, HttpUrl)
+ assert len(website.redirects) == 1
+ assert isinstance(website.redirects[0], HttpUrl)
+
+ # Test model_dump with mode="json" properly serializes HttpUrl to strings
+ dumped = website.model_dump(mode="json")
+ assert isinstance(dumped["url"], str)
+ assert dumped["url"] == "https://example.com/"
+ assert isinstance(dumped["redirects"][0], str)
+ assert dumped["redirects"][0] == "https://www.example.com/"
+
+
+def test_create_node_with_website(transform):
+ """Test that create_node can handle Website objects with HttpUrl fields."""
+ domain = Domain(domain="example.com")
+ website = Website(
+ url=HttpUrl("https://example.com"),
+ domain=domain,
+ active=True
+ )
+
+ # This should not raise an error about unsupported types
+ # The create_node method should properly serialize HttpUrl to string
+ try:
+ transform.create_node(website)
+ # If we get here without exception, the serialization works
+ success = True
+ except Exception as e:
+ success = False
+ error_msg = str(e)
+
+ # Note: This test won't actually create a node if Neo4j isn't connected,
+ # but it will verify the serialization doesn't fail
+ assert success or "Neo4j" in error_msg or "connection" in error_msg.lower()
+
+
+def test_create_relationship_with_website(transform):
+ """Test that create_relationship can handle Website objects with HttpUrl fields."""
+ domain = Domain(domain="example.com")
+ website = Website(
+ url=HttpUrl("https://example.com"),
+ domain=domain,
+ active=True
+ )
+
+ # This should not raise an error about unsupported HttpUrl types
+ # The create_relationship method should properly serialize HttpUrl to string
+ try:
+ transform.create_relationship(domain, website, "HAS_WEBSITE")
+ # If we get here without exception, the serialization works
+ success = True
+ except Exception as e:
+ success = False
+ error_msg = str(e)
+
+ # Note: This test won't actually create a relationship if Neo4j isn't connected,
+ # but it will verify the serialization doesn't fail
+ assert success or "Neo4j" in error_msg or "connection" in error_msg.lower()
+
+
+def test_extract_primary_value_with_httpurl(transform):
+ """Test that _extract_primary_value properly serializes HttpUrl."""
+ website = Website(
+ url=HttpUrl("https://example.com"),
+ domain=Domain(domain="example.com"),
+ active=True
+ )
+
+ # Extract the primary value (url field)
+ primary_value = transform._extract_primary_value(website)
+
+ # Should be a string, not an HttpUrl object
+ assert isinstance(primary_value, str)
+ assert primary_value == "https://example.com/"
+
+
+def test_preprocess_valid_domains(transform):
+ """Test preprocess with valid Domain objects."""
+ inputs = [
+ Domain(domain="example.com"),
+ Domain(domain="google.com"),
+ ]
+ result = transform.preprocess(inputs)
+
+ assert len(result) == 2
+ assert all(isinstance(item, Domain) for item in result)
+ assert result[0].domain == "example.com"
+ assert result[1].domain == "google.com"