A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat(transforms): support for Username and SocialAccount
dextmorgn1fb2c789143e
diff --git a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py b/flowsint-transforms/src/flowsint_transforms/individual/to_org.pyindex 9d376c3a..adb65499 100644--- a/flowsint-transforms/src/flowsint_transforms/individual/to_org.py+++ b/flowsint-transforms/src/flowsint_transforms/individual/to_org.py@@ -309,7 +309,7 @@ class IndividualToOrgTransform(Transform):# Create individual nodeself.create_node(- "Individual",+ "individual","full_name",individual_id,**individual.__dict__,@@ -323,7 +323,7 @@ class IndividualToOrgTransform(Transform):# Create relationship between individual and organizationself.create_relationship(- "Individual",+ "individual","full_name",individual_id,"Organization",diff --git a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.pyindex 4d241a0a..3f786914 100644--- a/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py+++ b/flowsint-transforms/src/flowsint_transforms/organization/to_infos.py@@ -324,7 +324,7 @@ class OrgToInfosTransform(Transform):"organization","org_id",org_key,- "Individual",+ "individual","full_name",dirigeant.full_name,"HAS_LEADER",diff --git a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.pyindex 29e53567..e288aab8 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_maigret.py@@ -4,7 +4,8 @@ from pathlib import Pathfrom typing import List, Unionfrom flowsint_core.utils import is_valid_usernamefrom flowsint_core.core.transform_base import Transform-from flowsint_types.social import SocialProfile+from flowsint_types import Username+from flowsint_types.social_account import SocialAccountfrom flowsint_core.core.logger import Loggerfalse_positives = ["LeagueOfLegends"]@@ -14,8 +15,8 @@ class MaigretTransform(Transform):"""[MAIGRET] Scans usernames for associated social accounts using Maigret."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[SocialProfile]- OutputType = List[SocialProfile]+ InputType = List[Username]+ OutputType = List[SocialAccount]@classmethoddef name(cls) -> str:@@ -34,13 +35,17 @@ class MaigretTransform(Transform):for item in data:obj = Noneif isinstance(item, str):- obj = SocialProfile(username=item)+ obj = Username(value=item)elif isinstance(item, dict) and "username" in item:- obj = SocialProfile(username=item["username"])- elif isinstance(item, SocialProfile):+ obj = Username(value=item["username"])+ elif isinstance(item, dict) and "value" in item:+ obj = Username(value=item["value"])+ elif isinstance(item, SocialAccount):+ obj = Username(value=item.username.value)+ obj = item.username.value+ elif isinstance(item, Username):obj = item-- if obj and obj.username and is_valid_username(obj.username):+ if obj and obj.value and is_valid_username(obj.value):cleaned.append(obj)return cleaned@@ -60,10 +65,8 @@ class MaigretTransform(Transform):)return output_file- def parse_maigret_output(- self, username: str, output_file: Path- ) -> List[SocialProfile]:- results: List[SocialProfile] = []+ def parse_maigret_output(self, username: str, output_file: Path) -> List[Username]:+ results: List[SocialAccount] = []if not output_file.exists():return results@@ -111,8 +114,8 @@ class MaigretTransform(Transform):followers = following = posts = Noneresults.append(- SocialProfile(- username=username,+ SocialAccount(+ username=Username(value=username),profile_url=profile_url,platform=platform,profile_picture_url=ids.get("image"),@@ -128,10 +131,10 @@ class MaigretTransform(Transform):async def scan(self, data: InputType) -> OutputType:results: OutputType = []for profile in data:- if not profile.username:+ if not profile.value:continue- output_file = self.run_maigret(profile.username)- parsed = self.parse_maigret_output(profile.username, output_file)+ output_file = self.run_maigret(profile.value)+ parsed = self.parse_maigret_output(profile.value, output_file)results.extend(parsed)return results@@ -142,40 +145,37 @@ class MaigretTransform(Transform):for profile in results:# Create social profile nodeself.create_node(- "social_profile",- "profile_url",+ "social_account",+ "platform",profile.profile_url,- username=profile.username,+ username=profile.username.value,platform=profile.platform,profile_picture_url=profile.profile_picture_url,bio=profile.bio,followers_count=profile.followers_count,following_count=profile.following_count,posts_count=profile.posts_count,- label=f"{profile.platform}:{profile.username}",- caption=f"{profile.platform}:{profile.username}",- color="#1DA1F2",- type="social_profile",+ label=f"{profile.username.value}",+ type="social_account",)-# Create username nodeself.create_node(- "username", "username", profile.username, username=profile.username+ "username", "value", profile.username.value, **profile.username.__dict__)# Create relationshipself.create_relationship("username",- "username",- profile.username,- "social_profile",- "profile_url",- profile.profile_url,+ "value",+ profile.username.value,+ "social_account",+ "platform",+ profile.platform,"HAS_SOCIAL_ACCOUNT",)self.log_graph_message(- f"{profile.username} -> account found on {profile.platform}"+ f"{profile.username.value} -> account found on {profile.platform}")return resultsdiff --git a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.pyindex 5081ca59..e90930d3 100644--- a/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py+++ b/flowsint-transforms/src/flowsint_transforms/social/to_sherlock.py@@ -2,7 +2,7 @@ import subprocessfrom pathlib import Pathfrom typing import List, Unionfrom flowsint_core.utils import is_valid_username-from flowsint_types.social import SocialProfile+from flowsint_types import SocialAccount, Usernamefrom flowsint_core.core.transform_base import Transformfrom flowsint_core.core.logger import Logger@@ -11,8 +11,8 @@ class SherlockTransform(Transform):"""[SHERLOCK] Scans the usernames for associated social accounts using Sherlock."""# Define types as class attributes - base class handles schema generation automatically- InputType = List[SocialProfile]- OutputType = List[SocialProfile]+ InputType = List[Username]+ OutputType = List[SocialAccount]@classmethoddef name(cls) -> str:@@ -31,13 +31,17 @@ class SherlockTransform(Transform):for item in data:obj = Noneif isinstance(item, str):- obj = SocialProfile(username=item)+ obj = Username(value=item)elif isinstance(item, dict) and "username" in item:- obj = SocialProfile(username=item["username"])- elif isinstance(item, SocialProfile):+ obj = Username(value=item["username"])+ elif isinstance(item, dict) and "value" in item:+ obj = Username(value=item["value"])+ elif isinstance(item, SocialAccount):+ obj = Username(value=item.username.value)+ obj = item.username.value+ elif isinstance(item, Username):obj = item-- if obj and obj.username and is_valid_username(obj.username):+ if obj and obj.value and is_valid_username(obj.value):cleaned.append(obj)return cleaned@@ -45,13 +49,12 @@ class SherlockTransform(Transform):"""Performs the scan using Sherlock on the list of usernames."""results: OutputType = []- for social in data:- username = social.username- output_file = Path(f"/tmp/sherlock_{username}.txt")+ for username in data:+ output_file = Path(f"/tmp/sherlock_{username.value}.txt")try:# Running the Sherlock command to perform the scanresult = subprocess.run(- ["sherlock", username, "-o", str(output_file)],+ ["sherlock", username.value, "-o", str(output_file)],capture_output=True,text=True,timeout=100,@@ -61,7 +64,7 @@ class SherlockTransform(Transform):Logger.error(self.sketch_id,{- "message": f"Sherlock failed for {username}: {result.stderr.strip()}"+ "message": f"Sherlock failed for {username.value}: {result.stderr.strip()}"},)continue@@ -70,7 +73,7 @@ class SherlockTransform(Transform):Logger.error(self.sketch_id,{- "message": f"Sherlock did not produce any output file for {username}."+ "message": f"Sherlock did not produce any output file for {username.value}."},)continue@@ -86,19 +89,19 @@ class SherlockTransform(Transform):# Create Social objects for each found accountfor platform, url in found_accounts.items():results.append(- SocialProfile(username=username, platform=platform, url=url)+ SocialAccount(username=username, platform=platform, url=url))except subprocess.TimeoutExpired:Logger.error(self.sketch_id,- {"message": f"Sherlock scan for {username} timed out."},+ {"message": f"Sherlock scan for {username.value} timed out."},)except Exception as e:Logger.error(self.sketch_id,{- "message": f"Unexpected error in Sherlock scan for {username}: {str(e)}"+ "message": f"Unexpected error in Sherlock scan for {username.value}: {str(e)}"},)@@ -109,18 +112,18 @@ class SherlockTransform(Transform):if not self.neo4j_conn:return results- for social in results:+ for social_account in results:self.create_node("social","username",- social.username,- platform=social.platform,- url=social.url,- caption=social.platform,+ social_account.username.value,+ platform=social_account.platform,+ url=social_account.url,+ caption=social_account.platform,type="social",)self.log_graph_message(- f"Found social account: {social.username} on {social.platform}"+ f"Found social account: {social_account.username.value} on {social_account.platform}")return resultsdiff --git a/flowsint-transforms/tests/transforms/social/maigret.py b/flowsint-transforms/tests/transforms/social/maigret.pyindex 3a47aef9..e70064e4 100644--- a/flowsint-transforms/tests/transforms/social/maigret.py+++ b/flowsint-transforms/tests/transforms/social/maigret.py@@ -1,6 +1,6 @@from pathlib import Pathfrom flowsint_transforms.socials.maigret import MaigretTransform-from flowsint_types.social import SocialProfile+from flowsint_types.social import SocialAccounttransform = MaigretTransform("sketch_123", "scan_123")@@ -12,15 +12,15 @@ def test_unprocessed_valid_usernames():]result = transform.preprocess(usernames)result_usernames = [d for d in result]- expected_usernames = [SocialProfile(username=d) for d in usernames]+ expected_usernames = [SocialAccount(username=d) for d in usernames]assert result_usernames == expected_usernamesdef test_preprocess_invalid_usernames():usernames = [- SocialProfile(username="toto123"),- SocialProfile(username="DorianXd78_Official"),- SocialProfile(username="This is not a username"),+ SocialAccount(username="toto123"),+ SocialAccount(username="DorianXd78_Official"),+ SocialAccount(username="This is not a username"),]result = transform.preprocess(usernames)@@ -34,7 +34,7 @@ def test_preprocess_multiple_formats():usernames = [{"username": "toto123"},{"invalid_key": "ValId_UseRnAme"},- SocialProfile(username="DorianXd78_Official"),+ SocialAccount(username="DorianXd78_Official"),"MySimpleUsername",]result = transform.preprocess(usernames)